[WDL 1.2] add task/workflow hints to grammar (only)

Author: mlin

WDL/_grammar.py

@@ -10,7 +10,7 @@
 )
 keywords["1.0"] = keywords["draft-2"] | set(["alias", "struct"])
 keywords["1.1"] = keywords["1.0"]
-keywords["1.2"] = keywords["1.1"] | set(["Directory", "env", "requirements"])
+keywords["1.2"] = keywords["1.1"] | set(["Directory", "env", "hints", "requirements"])
 keywords["development"] = keywords["1.2"]
 
 # Grammar versions and their definitions. The productions for WDL 1.2 and development will be
@@ -40,7 +40,7 @@
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
 workflow: "workflow" CNAME "{" workflow_element* "}"
-?workflow_element: input_decls | any_decl | call | scatter | conditional | workflow_outputs | meta_section
+?workflow_element: input_decls | any_decl | call | scatter | conditional | workflow_outputs | meta_section | hints_section
 
 scatter: "scatter" "(" CNAME "in" expr ")" "{" inner_workflow_element* "}"
 conditional: "if" "(" expr ")" "{" inner_workflow_element* "}"
@@ -65,6 +65,7 @@
              | output_decls
              | meta_section
              | requirements_section
+             | hints_section
              | task_env_decl -> noninput_decl
 
 tasks: task*
@@ -90,6 +91,14 @@
 requirements_section: ("requirements" | "runtime") "{" [runtime_kv (","? runtime_kv)*] "}"
 runtime_kv: CNAME ":" expr
 
+// hints section
+hints_section: hints_object
+hints_object: "hints" "{" [hint_kv (","? hint_kv)*] ","? "}"
+hint_kv: CNAME ":" hint_value
+?hint_value: expr | hints_object | io_hint
+io_hint: ("input" | "output") "{" [io_hint_kv ("," io_hint_kv)*] ","? "}"
+io_hint_kv: CNAME ("." CNAME)* ":" hint_value
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // decl
 ///////////////////////////////////////////////////////////////////////////////////////////////////

WDL/_parser.py

@@ -407,6 +407,9 @@ def requirements_section(self, meta, items):
             d[k] = v
         return {"runtime": d}
 
+    def hints_section(self, meta, items):
+        return {"hints": items[0]}
+
     def task(self, meta, items):
         d = {"noninput_decls": []}
         for item in items:
@@ -427,6 +430,7 @@ def task(self, meta, items):
                 assert "name" not in d
                 d["name"] = item.value
         self._check_keyword(self._sp(meta), d["name"])
+        # TODO: discarding d["hints"] for now (AST repr to be designed)
         return Tree.Task(
             self._sp(meta),
             d["name"],
@@ -532,6 +536,7 @@ def workflow(self, meta, items):
         output_idents_pos = None
         parameter_meta = None
         meta_section = None
+        hints_section = None
         for item in items[1:]:
             if isinstance(item, dict):
                 if "inputs" in item:
@@ -562,13 +567,20 @@ def workflow(self, meta, items):
                             self._sp(meta), "redundant workflow parameter_meta sections"
                         )
                     parameter_meta = item["parameter_meta"]
+                elif "hints" in item:
+                    if hints_section is not None:
+                        raise Error.MultipleDefinitions(
+                            self._sp(meta), "redundant workflow hints sections"
+                        )
+                    hints_section = item["hints"]
                 else:
                     assert False
             elif isinstance(item, (Tree.Call, Tree.Conditional, Tree.Decl, Tree.Scatter)):
                 elements.append(item)
             else:
                 assert False
         self._check_keyword(self._sp(meta), items[0].value)
+        # TODO: discarding hints_section for now (AST repr to be designed)
         return Tree.Workflow(
             self._sp(meta),
             items[0].value,

tests/spec_tests/config.yaml

@@ -34,18 +34,14 @@ wdl-1.2:
     - relative_and_absolute_task.wdl  # issue #214
     - test_struct.wdl                 # issue #728
     # To be implemented:
-    - allow_nested.wdl
     - chunk_array.wdl
     - file_sizes_task.wdl
     - multi_nested_inputs.wdl
-    - input_hint_task.wdl
     - join_paths_task.wdl
     - one_mount_point_task.wdl
     - string_to_file.wdl
-    - test_allow_nested_inputs.wdl
     - test_contains.wdl
     - test_find_task.wdl
-    - test_hints_task.wdl
     - test_keys.wdl
     - test_matches_task.wdl
     - test_runtime_info_task.wdl
@@ -63,7 +59,9 @@ wdl-1.2:
     - serde_pair.wdl                     # expected output is wrong
     - single_return_code_task.wdl        # issue #729
     - struct_to_struct.wdl               # typo capitalized Struct
+    - test_allow_nested_inputs.wdl       # two definitions of `greeting` in `task nested`
     - test_contains_key.wdl              # uses python-style `a if c else b` instead of `if c then a else b
+    - test_hints_task.wdl                # expected output is wrong (greetings.txt has no trailing newline, so wc -l returns 2 not 3)
     - test_length.wdl                    # typo in map literal
     - test_sub.wdl                       # issue #709
     - test_values.wdl                    # str_to_files should be str_to_ints

tests/test_1doc.py

@@ -551,6 +551,68 @@ def md5sum(filename, blocksize=65536):
         self.assertFalse(task.command.parts[0].strip().startswith("<<<"))
         self.assertFalse(task.command.parts[-1].strip().endswith(">>>"))
 
+    def test_hints(self):
+        txt = """
+        task t {
+            input {
+                Array[File] files
+            }
+            command <<<
+                set -euxo pipefail
+                mkdir files_out
+                find _miniwdl_inputs -type f -print0 | xargs -0 -iXXX cp XXX files_out/
+            >>>
+            output {
+                Array[File] files_out = glob("files_out/*")
+            }
+            runtime {
+                container: ["ubuntu:20.04"]
+            }
+            hints {
+                short_task: true
+                inputs: input {
+                    files: hints {
+                        localization_optional: true
+                    }
+                }
+            }
+        }
+        """
+        task = WDL.parse_tasks(txt, version="1.2")[0]
+        # TODO: test AST repr when implemented
+
+        txt = """
+        task t {
+            input {
+                Array[File] files
+            }
+            command <<<
+                set -euxo pipefail
+                mkdir files_out
+                find _miniwdl_inputs -type f -print0 | xargs -0 -iXXX cp XXX files_out/
+            >>>
+            output {
+                Array[File] files_out = glob("files_out/*")
+            }
+            runtime {
+                container: ["ubuntu:20.04"]
+            }
+            hints {
+                short_task: true
+                inputs: input {
+                    files: hints {
+                        localization_optional: true
+                    }
+                }
+            }
+            hints {
+                bogus: true
+            }
+        }
+        """
+        with self.assertRaises(WDL.Error.MultipleDefinitions):
+            WDL.parse_tasks(txt, version="1.2")[0].typecheck()
+
 
 class TestTypes(unittest.TestCase):
     def test_parser(self):
@@ -2515,6 +2577,49 @@ def test_keywords(self):
                 self.assertIsInstance(err.pos.line, int)
                 self.assertIsInstance(err.pos.column, int)
 
+    def test_workflow_hints(self):
+        doc = r"""version 1.2
+        workflow w {
+            input {
+                Array[String] names
+            }
+            scatter (name in names) {
+                String greeting = "Hello, " + name + "!"
+            }
+            output {
+                Array[String] greetings = greeting
+            }
+            hints {
+                allow_nested_inputs: true
+            }
+        }
+        """
+        doc = WDL.parse_document(doc)
+        doc.typecheck()
+        # TODO: check hints AST once implemented
+
+        doc = r"""version 1.2
+        workflow w {
+            input {
+                Array[String] names
+            }
+            scatter (name in names) {
+                String greeting = "Hello, " + name + "!"
+            }
+            output {
+                Array[String] greetings = greeting
+            }
+            hints {
+                allow_nested_inputs: true
+            }
+            hints {
+                bogus: true
+            }
+        }
+        """
+        with self.assertRaises(WDL.Error.MultipleDefinitions):
+            WDL.parse_document(doc)
+
 class TestNoneLiteral(unittest.TestCase):
     def test_none_expr(self):
         doc = r"""

tests/test_7runner.py

@@ -882,6 +882,9 @@ def test_weird_filenames(self):
             output {
                 Array[File] files_out = t.files_out
             }
+            hints {
+                allow_nested_inputs: true
+            }
         }
 
         task t {
@@ -899,6 +902,14 @@ def test_weird_filenames(self):
             runtime {
                 container: ["ubuntu:20.04"]
             }
+            hints {
+                short_task: true
+                inputs: input {
+                    files: hints {
+                        localization_optional: true
+                    }
+                }
+            }
         }
         """