Hints serverside

src/dirac_cwl_proto/execution_hooks/core.py

@@ -372,3 +372,10 @@ class TransformationExecutionHooksHint(ExecutionHooksHint):
     group_size: Optional[Dict[str, int]] = Field(
         default=None, description="Input grouping configuration for transformation jobs"
     )
+
+    # TODO: I can't use TransformationExecutionHooksHint.fromcwl
+    #  normally without doing that..? (mypi won't let me commit
+    #  since he thinks it's a ExecutionHooksHint)
+    @classmethod
+    def from_cwl(cls, cwl_object: Any) -> "TransformationExecutionHooksHint":
+        return cls(**super().from_cwl(cwl_object).dict())

src/dirac_cwl_proto/submission_models.py

@@ -73,8 +73,6 @@ class TransformationSubmissionModel(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     task: CommandLineTool | Workflow | ExpressionTool
-    execution_hooks: TransformationExecutionHooksHint
-    scheduling: SchedulingHint
 
     @field_serializer("task")
     def serialize_task(self, value):

src/dirac_cwl_proto/transformation/__init__.py

@@ -14,7 +14,6 @@
 from cwl_utils.parser.cwl_v1_2 import File
 from rich import print_json
 from rich.console import Console
-from ruamel.yaml import YAML
 from schema_salad.exceptions import ValidationException
 
 from dirac_cwl_proto.execution_hooks import (
@@ -40,9 +39,6 @@
 @app.command("submit")
 def submit_transformation_client(
     task_path: str = typer.Argument(..., help="Path to the CWL file"),
-    metadata_path: Optional[str] = typer.Option(
-        None, help="Path to metadata file used to generate the input query"
-    ),
     # Specific parameter for the purpose of the prototype
     local: Optional[bool] = typer.Option(
         True, help="Run the jobs locally instead of submitting them to the router"
@@ -73,22 +69,7 @@ def submit_transformation_client(
         return typer.Exit(code=1)
     console.print(f"\t[green]:heavy_check_mark:[/green] Task {task_path}")
 
-    # Load the metadata: at this stage, only the structure is validated, not the content
-    metadata_model = TransformationExecutionHooksHint()
-    if metadata_path:
-        with open(metadata_path, "r") as file:
-            metadata = YAML(typ="safe").load(file)
-        metadata_model = TransformationExecutionHooksHint(**metadata)
-    console.print("\t[green]:heavy_check_mark:[/green] Metadata")
-
-    transformation_scheduling = SchedulingHint.from_cwl(task)
-    console.print("\t[green]:heavy_check_mark:[/green] Description")
-
-    transformation = TransformationSubmissionModel(
-        task=task,
-        execution_hooks=metadata_model,
-        scheduling=transformation_scheduling,
-    )
+    transformation = TransformationSubmissionModel(task=task)
     console.print(
         "[green]:heavy_check_mark:[/green] [bold]CLI:[/bold] Transformation validated."
     )
@@ -134,20 +115,32 @@ def submit_transformation_router(transformation: TransformationSubmissionModel)
     # - if there is no execution_hooks, the transformation is not waiting for an input and can go on
     # - if there is execution_hooks, the transformation is waiting for an input
     job_model_params = []
+
+    try:
+        (
+            transformation_execution_hooks,
+            transformation_scheduling_hints,
+        ) = (
+            TransformationExecutionHooksHint.from_cwl(transformation.task),
+            SchedulingHint.from_cwl(transformation.task),
+        )
+    except Exception as exc:
+        raise ValueError(f"Invalid DIRAC hints:\n{exc}") from exc
+
     if (
-        transformation.execution_hooks.configuration
-        and transformation.execution_hooks.group_size
+        transformation_execution_hooks.configuration
+        and transformation_execution_hooks.group_size
     ):
         # Get the metadata class
-        transformation_metadata = transformation.execution_hooks.to_runtime(
+        transformation_metadata = transformation_execution_hooks.to_runtime(
             transformation
         )
 
         # Build the input cwl for the jobs to submit
         logger.info("Getting the input data for the transformation...")
         input_data_dict = {}
         min_length = None
-        for input_name, group_size in transformation.execution_hooks.group_size.items():
+        for input_name, group_size in transformation_execution_hooks.group_size.items():
             # Get input query
             logger.info(f"\t- Getting input query for {input_name}...")
             input_query = transformation_metadata.get_input_query(input_name)
@@ -177,8 +170,8 @@ def submit_transformation_router(transformation: TransformationSubmissionModel)
     jobs = JobSubmissionModel(
         task=transformation.task,
         parameters=job_model_params,
-        scheduling=transformation.scheduling,
-        execution_hooks=transformation.execution_hooks,
+        scheduling=transformation_scheduling_hints,
+        execution_hooks=transformation_execution_hooks,
     )
     logger.info("Jobs built!")
 

test/test_workflows.py

@@ -293,31 +293,19 @@ def test_run_job_validation_failure(
         ("test/workflows/crypto/md5.cwl", None),
         # --- Pi example ---
         # There is no input expected
-        (
-            "test/workflows/pi/pisimulate.cwl",
-            "test/workflows/pi/type_dependencies/transformation/metadata-pi_simulate.yaml",
-        ),
+        ("test/workflows/pi/pisimulate.cwl", None),
         # --- Pi v2 example ---
         # There is no input expected
-        (
-            "test/workflows/merge/pisimulate_v2.cwl",
-            "test/workflows/merge/type_dependencies/transformation/metadata-pi_simulate_v2.yaml",
-        ),
+        ("test/workflows/merge/pisimulate_v2.cwl", None),
         # --- LHCb example ---
-        (
-            "test/workflows/lhcb/lhcbsimulate.cwl",
-            "test/workflows/lhcb/type_dependencies/transformation/metadata-lhcb_simulate.yaml",
-        ),
+        ("test/workflows/lhcb/lhcbsimulate.cwl", None),
         # --- Mandelbrot example ---
-        (
-            "test/workflows/mandelbrot/image-prod.cwl",
-            "test/workflows/mandelbrot/type_dependencies/transformation/metadata-mandelbrot_imageprod.yaml",
-        ),
+        ("test/workflows/mandelbrot/image-prod.cwl", None),
         # --- Gaussian fit example ---
         # Data generation workflow
         (
             "test/workflows/gaussian_fit/data_generation/data-generation-workflow.cwl",
-            "test/workflows/gaussian_fit/type_dependencies/transformation/inputs-data-generation.yaml",
+            None,
         ),
     ],
 )
@@ -343,7 +331,7 @@ def test_run_nonblocking_transformation_success(
         # --- Pi example ---
         (
             "test/workflows/pi/pigather.cwl",
-            "test/workflows/pi/type_dependencies/transformation/metadata-pi_gather.yaml",
+            None,
             {
                 "filecatalog/pi/100": [
                     "test/workflows/pi/type_dependencies/job/result_1.sim",
@@ -357,7 +345,7 @@ def test_run_nonblocking_transformation_success(
         # --- LHCb example ---
         (
             "test/workflows/lhcb/lhcbreconstruct.cwl",
-            "test/workflows/lhcb/type_dependencies/transformation/metadata-lhcb_reconstruct.yaml",
+            None,
             {
                 "filecatalog/lhcb/456/123/simulation": [
                     "test/workflows/lhcb/type_dependencies/job/Gauss_123_456_1.sim",
@@ -369,7 +357,7 @@ def test_run_nonblocking_transformation_success(
         # --- Mandelbrot example ---
         (
             "test/workflows/mandelbrot/image-merge.cwl",
-            "test/workflows/mandelbrot/type_dependencies/transformation/metadata-mandelbrot_imagemerge.yaml",
+            None,
             {
                 "filecatalog/mandelbrot/images/raw/1920x1080/": [
                     "test/workflows/mandelbrot/type_dependencies/transformation/data_1.txt",
@@ -381,7 +369,7 @@ def test_run_nonblocking_transformation_success(
         # Gaussian fit workflow
         (
             "test/workflows/gaussian_fit/gaussian_fit/gaussian-fit-workflow.cwl",
-            "test/workflows/gaussian_fit/type_dependencies/transformation/inputs-gaussian-fit.yaml",
+            None,
             {
                 "filecatalog/gaussian_fit/data-generation-1/": [
                     "test/workflows/gaussian_fit/type_dependencies/transformation/data-generation-1/data_gen1.txt",

test/workflows/gaussian_fit/data_generation/data-generation-workflow.cwl

@@ -4,6 +4,9 @@ label: "Data Generation Workflow"
 doc: >
   This workflow generates data using two independent data-generation tools.
 
+hints:
+  $import: "../type_dependencies/transformation/inputs-data-generation.yaml"
+
 inputs:
   output_file_name_1:
     type: string

test/workflows/gaussian_fit/gaussian_fit/gaussian-fit-workflow.cwl

@@ -8,6 +8,9 @@ requirements:
   SubworkflowFeatureRequirement: {}
   MultipleInputFeatureRequirement: {}
 
+hints:
+  $import: "../type_dependencies/transformation/inputs-gaussian-fit.yaml"
+
 inputs:
   data1:
     type: File[]

test/workflows/gaussian_fit/type_dependencies/transformation/inputs-data-generation.yaml

@@ -1,4 +1,5 @@
-hook_plugin: DataGenerationPlugin
-configuration:
-  output_file_name_1: data-gen1.txt
-  output_file_name_2: data-gen2.txt
+dirac:execution-hooks:
+  hook_plugin: DataGenerationPlugin
+  configuration:
+    output_file_name_1: data-gen1.txt
+    output_file_name_2: data-gen2.txt

test/workflows/gaussian_fit/type_dependencies/transformation/inputs-gaussian-fit.yaml

@@ -1,7 +1,8 @@
-hook_plugin: GaussianFitPlugin
-group_size:
-  data1: 1
-  data2: 1
-configuration:
-  data1: ["data-gen1.txt"]
-  data2: ["data-gen2.txt"]
+dirac:execution-hooks:
+  hook_plugin: GaussianFitPlugin
+  group_size:
+    data1: 1
+    data2: 1
+  configuration:
+    data1: ["data-gen1.txt"]
+    data2: ["data-gen2.txt"]

test/workflows/lhcb/lhcbreconstruct.cwl

@@ -8,6 +8,9 @@ requirements:
     ramMin: 2048
     ramMax: 4096
 
+hints:
+  $import: "type_dependencies/transformation/metadata-lhcb_reconstruct.yaml"
+
 inputs:
   run-id:
     type: int

test/workflows/lhcb/lhcbsimulate.cwl

@@ -43,6 +43,9 @@ requirements:
     coresMin: 1
     ramMin: 2048
 
+hints:
+  $import: "type_dependencies/transformation/metadata-lhcb_simulate.yaml"
+
 inputs:
   run-id:
     type: int

test/workflows/lhcb/type_dependencies/transformation/metadata-lhcb_reconstruct.yaml

@@ -1,6 +1,7 @@
-hook_plugin: LHCbReconstructionPlugin
-group_size:
-  files: 3
-configuration:
-  run-id: 123
-  task-id: 456
+dirac:execution-hooks:
+  hook_plugin: LHCbReconstructionPlugin
+  group_size:
+    files: 3
+  configuration:
+    run-id: 123
+    task-id: 456

test/workflows/lhcb/type_dependencies/transformation/metadata-lhcb_simulate.yaml

@@ -1 +1,2 @@
-hook_plugin: LHCbSimulationPlugin
+dirac:execution-hooks:
+  hook_plugin: LHCbSimulationPlugin

test/workflows/mandelbrot/image-merge.cwl

@@ -14,6 +14,9 @@ outputs:
     type: File[]?
     outputSource: merge-mandelbrot/log
 
+hints:
+  $import: "type_dependencies/transformation/metadata-mandelbrot_imagemerge.yaml"
+
 steps:
   # Clone the project
   get-mandelbrot:

test/workflows/mandelbrot/image-prod.cwl

@@ -40,6 +40,9 @@ outputs:
     type: File[]?
     outputSource: run-mandelbrot/log
 
+hints:
+  $import: "type_dependencies/transformation/metadata-mandelbrot_imageprod.yaml"
+
 steps:
   # Clone the project
   get-mandelbrot:

test/workflows/mandelbrot/type_dependencies/transformation/metadata-mandelbrot_imagemerge.yaml

@@ -1,13 +1,14 @@
-hook_plugin: MandelBrotMergingPlugin
-group_size:
-  data: 3
-configuration:
-  precision: 0.0005
-  max_iterations: 1000
-  start_x: -0.5
-  start_y: 0.0
-  step: 1
-  split: 1
-  width: 1920
-  height: 1080
-  output_name: "data_"
+dirac:execution-hooks:
+  hook_plugin: MandelBrotMergingPlugin
+  group_size:
+    data: 3
+  configuration:
+    precision: 0.0005
+    max_iterations: 1000
+    start_x: -0.5
+    start_y: 0.0
+    step: 1
+    split: 1
+    width: 1920
+    height: 1080
+    output_name: "data_"

test/workflows/mandelbrot/type_dependencies/transformation/metadata-mandelbrot_imageprod.yaml

@@ -1 +1,2 @@
-hook_plugin: MandelBrotGenerationPlugin
+dirac:execution-hooks:
+  hook_plugin: MandelBrotGenerationPlugin

test/workflows/merge/pisimulate_v2.cwl

@@ -6,6 +6,9 @@ requirements:
     coresMin: 2
     ramMin: 1024
 
+hints:
+  $import: "type_dependencies/transformation/metadata-pi_simulate_v2.yaml"
+
 inputs:
   num-points:
     type: int

test/workflows/merge/type_dependencies/transformation/metadata-pi_gather.yaml

@@ -1,4 +1,5 @@
-hook_plugin: PiGather
-configuration:
-  num_points: 1000
-  input-data: 2
+dirac:execution-hooks:
+  hook_plugin: PiGather
+  configuration:
+    num_points: 1000
+    input-data: 2

test/workflows/merge/type_dependencies/transformation/metadata-pi_simulate_v2.yaml

@@ -1 +1,2 @@
-hook_plugin: PiSimulateV2Plugin
+dirac:execution-hooks:
+  hook_plugin: PiSimulateV2Plugin

test/workflows/pi/pigather.cwl

@@ -6,6 +6,9 @@ requirements:
     coresMin: 1
     ramMin: 1024
 
+hints:
+  $import: "type_dependencies/transformation/metadata-pi_gather.yaml"
+
 inputs:
   input-data:
     type: File[]

test/workflows/pi/pisimulate.cwl

@@ -6,6 +6,9 @@ requirements:
     coresMin: 2
     ramMin: 1024
 
+hints:
+  $import: "type_dependencies/transformation/metadata-pi_simulate.yaml"
+
 inputs:
   num-points:
     type: int

test/workflows/pi/type_dependencies/transformation/metadata-pi_gather.yaml

@@ -1,5 +1,6 @@
-hook_plugin: PiGatherPlugin
-group_size:
-  input-data: 5
-configuration:
-  num_points: 100
+dirac:execution-hooks:
+  hook_plugin: PiGatherPlugin
+  group_size:
+    input-data: 5
+  configuration:
+    num_points: 100

test/workflows/pi/type_dependencies/transformation/metadata-pi_simulate.yaml

@@ -1,3 +1,4 @@
 # Override hints from CWL workflow
-configuration:
-  num_points: 2000  # Override default 1000 from CWL
+dirac:execution-hooks:
+  configuration:
+    num_points: 2000  # Override default 1000 from CWL