Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,7 @@ multineedle_simple_agent:
type: train
license: Apache 2.0
jsonl_fpath: resources_servers/multineedle/data/train.jsonl
num_repeats: 1
gitlab_identifier:
dataset_name: multineedle
version: 0.0.1
Expand All @@ -623,6 +624,7 @@ multineedle_simple_agent:
type: validation
license: Apache 2.0
jsonl_fpath: resources_servers/multineedle/data/validation.jsonl
num_repeats: 1
Comment thread
mahanfathi marked this conversation as resolved.
Resolved
gitlab_identifier:
dataset_name: multineedle
version: 0.0.1
Expand All @@ -631,12 +633,14 @@ multineedle_simple_agent:
- name: example
type: example
jsonl_fpath: resources_servers/multineedle/data/example.jsonl
num_repeats: 1
```

A dataset object consists of:
- Name: An identifier for you
- Type: train, validation, or example. Train and validation are as used in NeMo RL or other train frameworks. More information about the example type is in the next section.
- Jsonl fpath: the local file path to your jsonl file for this dataset.
- Num repeats: optionally repeat each row when preparing or collating data. Defaults to 1 if unspecified.
- Gitlab identifier: The remote path to the dataset as held in the Gitlab dataset registry. This field is required for train and validation datasets. (Not required for example datasets since those are required to be committed to Git).
- License: The license of that dataset. Required for train and validation datasets and not required for example datasets, similar in principle to the Gitlab identifier.
- Start idx, end idx: used for slicing your dataset.
Expand Down
3 changes: 3 additions & 0 deletions nemo_gym/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,7 @@ def init_resources_server(): # pragma: no cover
- name: train
type: train
jsonl_fpath: resources_servers/{server_type_name}/data/train.jsonl
num_repeats: 1
gitlab_identifier:
dataset_name: {server_type_name}
version: 0.0.1
Expand All @@ -526,6 +527,7 @@ def init_resources_server(): # pragma: no cover
- name: validation
type: validation
jsonl_fpath: resources_servers/{server_type_name}/data/validation.jsonl
num_repeats: 1
gitlab_identifier:
dataset_name: {server_type_name}
version: 0.0.1
Expand All @@ -534,6 +536,7 @@ def init_resources_server(): # pragma: no cover
- name: example
type: example
jsonl_fpath: resources_servers/{server_type_name}/data/example.jsonl
num_repeats: 1
""")

app_fpath = dirpath / "app.py"
Expand Down
1 change: 1 addition & 0 deletions nemo_gym/config_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ class DatasetConfig(BaseModel):
type: DatasetType
jsonl_fpath: str

num_repeats: int = Field(default=1, ge=1)
gitlab_identifier: Optional[JsonlDatasetGitlabIdentifer] = None
license: Optional[
Union[
Expand Down
30 changes: 19 additions & 11 deletions nemo_gym/train_data_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import json
from abc import abstractmethod
from collections import Counter, defaultdict
from itertools import count, repeat
from math import sqrt
from pathlib import Path
from shutil import copyfileobj
Expand Down Expand Up @@ -460,20 +459,29 @@ def _validate_samples_and_aggregate_metrics_single_sample(

aggregate_other_metrics(state.other_metrics, sample_dict)

def _iter_dataset_lines(self, dataset_config: DatasetConfig):
repeats = dataset_config.num_repeats

# Print dataset repetition info
if repeats > 1:
print(
f"Dataset {dataset_config.name} repeating {repeats}x: each line repeated {repeats} times (e.g. pattern: abc -> aaaabbbbcccc)"
)

# Don't load everything into memory at once. Throw things away immediately.
with open(dataset_config.jsonl_fpath) as f:
for line in f:
for _ in range(repeats):
yield line

def _validate_samples_and_aggregate_metrics_single_dataset(
self, dataset_config: DatasetConfig
) -> DatasetValidatorState:
state = DatasetValidatorState()

map_fn = self._validate_samples_and_aggregate_metrics_single_sample
with open(dataset_config.jsonl_fpath) as f:
# Don't load everything into memory at once. Throw things away immediately.
Comment thread
mahanfathi marked this conversation as resolved.
Resolved
any(
tqdm(
map(map_fn, repeat(state), count(), f),
desc=f"Process {dataset_config.jsonl_fpath}",
)
)
for sample_idx, sample_dict_str in enumerate(self._iter_dataset_lines(dataset_config)):
map_fn(state, sample_idx, sample_dict_str)

postprocess_other_metrics(state.metrics, state.other_metrics)

Expand Down Expand Up @@ -548,8 +556,8 @@ def _collate_samples_single_type(

data_path = Path(d.jsonl_fpath)
prepare_path = data_path.with_name(f"{data_path.stem}_prepare.jsonl")
with open(data_path) as source, open(prepare_path, "w") as target:
for line in tqdm(source, desc=f"Preparing data at {data_path}"):
with open(prepare_path, "w") as target:
for line in self._iter_dataset_lines(d):
d = json.loads(line)
d[AGENT_REF_KEY] = AgentServerRef(type="responses_api_agents", name=c.name).model_dump()
target.write(f"{json.dumps(d)}\n")
Expand Down
1 change: 1 addition & 0 deletions resources_servers/multineedle/data/example_metrics.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": null,
"license": null,
"Number of examples": 5,
Expand Down
172 changes: 171 additions & 1 deletion tests/unit_tests/test_train_data_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@
from pathlib import Path
from unittest.mock import MagicMock, mock_open

from pydantic import ValidationError
from pytest import MonkeyPatch, raises

import nemo_gym.global_config
import nemo_gym.train_data_utils
from nemo_gym.config_types import ResponsesAPIAgentServerInstanceConfig
from nemo_gym.config_types import DatasetConfig, ResponsesAPIAgentServerInstanceConfig
from nemo_gym.global_config import DictConfig, GlobalConfigDictParser
from nemo_gym.train_data_utils import (
AvgMinMax,
Expand Down Expand Up @@ -79,6 +80,7 @@ def test_load_and_validate_server_instance_configs_sanity(self, monkeypatch: Mon
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -121,6 +123,7 @@ def test_load_datasets_sanity(self) -> None:
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -166,6 +169,7 @@ def test_load_datasets_missing_example_dataset_raises_AssertionError(self) -> No
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example_missing.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -281,6 +285,7 @@ def custom_open(filename, mode="r"):
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -421,6 +426,7 @@ def custom_open(filename, mode="r"):
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -522,6 +528,168 @@ def test_validate_samples_and_aggregate_metrics_single_sample(self) -> None:
assert expected_metrics.model_dump() == state.metrics.model_dump()


class TestIterDatasetLines:
def test_iter_dataset_lines_num_repeats(self, tmp_path):
"""Test _iter_dataset_lines with different num_repeats values"""
processor = TrainDataProcessor()

# Create a test dataset file
test_file = tmp_path / "test_data.jsonl"
test_data = ['{"id": 1, "content": "line1"}', '{"id": 2, "content": "line2"}']
test_file.write_text("\n".join(test_data) + "\n")

# Test default behavior (num_repeats defaults to 1 when not specified)
config = DatasetConfig(name="test", type="example", jsonl_fpath=str(test_file))
lines = list(processor._iter_dataset_lines(config))
expected = ['{"id": 1, "content": "line1"}\n', '{"id": 2, "content": "line2"}\n']
assert lines == expected

# Test num_repeats=1 (explicit)
config = DatasetConfig(name="test", type="example", jsonl_fpath=str(test_file), num_repeats=1)
lines = list(processor._iter_dataset_lines(config))
assert lines == expected

# Test num_repeats=3
config = DatasetConfig(name="test", type="example", jsonl_fpath=str(test_file), num_repeats=3)
lines = list(processor._iter_dataset_lines(config))
expected_repeated = [
'{"id": 1, "content": "line1"}\n',
'{"id": 1, "content": "line1"}\n',
'{"id": 1, "content": "line1"}\n',
'{"id": 2, "content": "line2"}\n',
'{"id": 2, "content": "line2"}\n',
'{"id": 2, "content": "line2"}\n',
]
assert lines == expected_repeated


class TestDatasetConfigNumRepeats:
def test_dataset_config_num_repeats_valid_values(self):
"""Test DatasetConfig with valid num_repeats values"""

# Test default (when not specified, defaults to 1)
config = DatasetConfig(name="test", type="example", jsonl_fpath="test.jsonl")
assert config.num_repeats == 1

# Test valid positive integers
for repeats in [1, 2, 5, 10, 100]:
config = DatasetConfig(name="test", type="example", jsonl_fpath="test.jsonl", num_repeats=repeats)
assert config.num_repeats == repeats

def test_dataset_config_num_repeats_invalid_values(self):
"""Test DatasetConfig with invalid num_repeats values"""

# Test zero
with raises(ValidationError, match="Input should be greater than or equal to 1"):
DatasetConfig(name="test", type="example", jsonl_fpath="test.jsonl", num_repeats=0)

# Test negative values
with raises(ValidationError, match="Input should be greater than or equal to 1"):
DatasetConfig(name="test", type="example", jsonl_fpath="test.jsonl", num_repeats=-1)


class TestNumRepeatsMetricsAggregation:
def test_validate_samples_with_num_repeats(self, tmp_path):
"""Test that metrics and sample enumeration work correctly with num_repeats"""
processor = TrainDataProcessor()

# Test with valid samples and num_repeats=2
test_file = tmp_path / "test_data.jsonl"
test_data = [
'{"responses_create_params": {"input": [{"role": "user", "content": "test1"}]}, "temperature": 0.5}',
'{"invalid": "sample"}', # Invalid sample for testing enumeration
]
test_file.write_text("\n".join(test_data) + "\n")

config = DatasetConfig(name="test", type="example", jsonl_fpath=str(test_file), num_repeats=2)
state = processor._validate_samples_and_aggregate_metrics_single_dataset(config)

# Should process 4 samples total (2 lines * 2 repeats), but only valid ones count in metrics
# The metrics count only tracks successful processing (only 2 successful out of 4 total)
assert state.metrics.number_of_examples == 2
# Indices 2,3 should be offending (both repeats of the invalid sample)
assert state.offending_example_idxs == [2, 3]


class TestNumRepeatsDataPreparation:
def test_prepare_samples_with_num_repeats(self, tmp_path, monkeypatch: MonkeyPatch):
"""Test that data preparation correctly repeats samples based on num_repeats"""
write_filenames_to_mock = dict()
original_open = open

def custom_open(filename, mode="r"):
if mode in ["w", "wb"]:
write_filenames_to_mock[filename] = mock_open()
return write_filenames_to_mock[filename]()
if filename in write_filenames_to_mock:
write_mock: MagicMock = write_filenames_to_mock[filename]
written_data = write_mock.return_value.write.call_args_list[0].args[0]
return mock_open(read_data=written_data)()
return original_open(filename, mode)

monkeypatch.setattr("builtins.open", custom_open)

processor = TrainDataProcessor()
test_file = tmp_path / "test_data.jsonl"
test_data = [
'{"responses_create_params": {"input": [{"role": "user", "content": "test1"}]}}',
'{"responses_create_params": {"input": [{"role": "user", "content": "test2"}]}}',
]
test_file.write_text("\n".join(test_data) + "\n")

config = TrainDataProcessorConfig(output_dirpath="", mode="example_validation", should_download=False)
server_type_config_dict = {
"responses_api_agents": {
"simple_agent": {
"host": "127.0.0.1",
"port": 12345,
"entrypoint": "app.py",
"datasets": [
{
"name": "example",
"type": "example",
"jsonl_fpath": str(test_file),
"num_repeats": 3,
"gitlab_identifier": None,
"license": None,
}
],
"resources_server": {"type": "resources_servers", "name": "test_resources_server"},
"model_server": {"type": "responses_api_models", "name": "policy_model"},
}
}
}
server_config = ResponsesAPIAgentServerInstanceConfig(
name="test_agent",
server_type_config_dict=DictConfig(server_type_config_dict),
responses_api_agents=server_type_config_dict["responses_api_agents"],
)

processor.collate_samples(
config=config,
server_instance_configs=[server_config],
dataset_type_to_aggregate_metrics={
"example": DatasetMetrics(
is_aggregated=False,
number_of_examples=6,
number_of_tools=AvgMinMax(is_aggregated=False, total=6, average=0.0, min=0.0, max=0.0),
json_dumped_number_of_words=AvgMinMax(
is_aggregated=False, total=6, average=100.0, min=50.0, max=150.0
),
number_of_turns=AvgMinMax(is_aggregated=False, total=6, average=1.0, min=1.0, max=1.0),
temperature=AvgMinMax(
is_aggregated=False, total=0, average=0, min=float("inf"), max=float("-inf")
),
)
},
)

# Verify 6 writes occurred (2 lines * 3 repeats each)
expected_prepare_file = test_file.with_name(f"{test_file.stem}_prepare.jsonl")
prepare_mock = write_filenames_to_mock[expected_prepare_file]
assert len(prepare_mock.return_value.write.call_args_list) == 6


class TestCollateSamples:
def test_collate_samples_sanity(self, monkeypatch: MonkeyPatch) -> None:
write_filenames_to_mock = dict()
Expand Down Expand Up @@ -559,6 +727,7 @@ def custom_open(filename, mode="r"):
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down Expand Up @@ -655,6 +824,7 @@ def custom_open(filename, mode="r"):
"name": "example",
"type": "example",
"jsonl_fpath": "resources_servers/multineedle/data/example.jsonl",
"num_repeats": 1,
"gitlab_identifier": None,
"license": None,
}
Expand Down