provide more docstrings

Signed-off-by: Oleg Silkin <[email protected]>

Author: RobotSail

processed-data/test-dataprocess.py renamed to examples/test-dataprocess.py

@@ -1,14 +1,16 @@
-# Standard
-import argparse
-
-# First Party
-from instructlab.training.data_process import process_data
+# SPDX-License-Identifier: Apache-2.0
 
 """
 This file showcases how someone can use the data-processing script to
 take a dataset from `messages` format into raw input_ids and labels
 """
 
+# Standard
+import argparse
+
+# First Party
+from instructlab.training.data_process import process_data
+
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--data-path", type=str, required=True)
@@ -38,7 +40,6 @@
     data_path=args.data_path,
     data_output_path=args.data_output_path,
     model_path=model_path,
-    use_legacy_method=args.legacy,
     chat_tmpl_path=args.chat_tmpl_path,
     max_seq_len=args.max_seq_len,
     num_cpu_procs=1,

src/instructlab/training/config.py

@@ -170,12 +170,6 @@ class TrainingArgs(BaseModel):
     # after we have tokenized it
     data_output_dir: str
 
-    # New option to use the new data processor implementation
-    use_legacy_data_processor: bool = Field(
-        default=False,
-        description="this field determines if the legacy data processor should be used instead",
-    )
-
     max_seq_len: int
     max_batch_len: int
     num_epochs: int
@@ -220,3 +214,12 @@ class TrainingArgs(BaseModel):
     # will overwrite the previous checkpoint directory, keeping only one directory called
     # "last_epoch". This works alongside the '--checkpoint_at_epoch' flag.
     keep_last_checkpoint_only: Optional[bool] = False
+
+    # TODO(osilkin):
+    #   we are only exposing this here because `run_training` today is implicitly coupled
+    #   with `process_data`. Since we don't have a specific field for data processing arguments,
+    #   we are forced to expose this. We should uncouple training from data processing and remove this.
+    data_process_num_cpu_procs: int = Field(
+        default=16,
+        description="This is the number of processes used for multiprocessing when processing the data",
+    )