Merge branch 'main' into mlin/hints-grammar

Author: mlin

.github/workflows/CI.yml

@@ -15,15 +15,20 @@ jobs:
           # disabled due to test data bitrot (Apr 2023)
           # - skylab_bulk_rna
     runs-on: ubuntu-22.04
+    env:
+      MAXIMIZE_SPACE_TARGETS: '["viral_assemble","viral_refbased"]'
     steps:
     - name: Maximize build space
-      uses: AdityaGarg8/remove-unwanted-software@v3
+      uses: AdityaGarg8/remove-unwanted-software@v5
       with:
         remove-dotnet: 'true'
         remove-android: 'true'
         remove-haskell: 'true'
         remove-codeql: 'true'
-      if: ${{ matrix.ci_target == 'viral_assemble' }}
+        remove-docker-images: 'true'
+        remove-large-packages: 'true'
+        remove-cached-tools: 'true'
+      if: ${{ contains(fromJson(env.MAXIMIZE_SPACE_TARGETS), matrix.ci_target) }}
     - name: Login to GitHub Packages Docker Registry
       uses: docker/login-action@v1
       with:

README.md

@@ -56,3 +56,9 @@ Feedback and contributions to miniwdl are welcome, via issues and pull requests
 ## Security
 
 Please disclose security issues responsibly by contacting [email protected].
+
+## Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code. 
+Please report unacceptable behavior to [[email protected]](mailto:[email protected]).

SECURITY.md

@@ -0,0 +1,2 @@
+## Reporting Security Issues
+If you believe you have found a security issue, please responsibly disclose by contacting us at [[email protected]](mailto:[email protected]).

WDL/CLI.py

@@ -142,16 +142,13 @@ def __call__(self, parser, namespace, values, option_string=None):
         else:
             print("miniwdl version unknown")
 
-        # show plugin versions
-        # importlib_metadata doesn't seem to provide EntryPoint.dist to get from an entry point to
-        # the metadata of the package providing it; continuing to use pkg_resources for this. Risk
-        # that they give inconsistent results?
-        import pkg_resources  # type: ignore
+        import importlib_metadata
 
         for group in runtime.config.default_plugins().keys():
             group = f"miniwdl.plugin.{group}"
-            for plugin in pkg_resources.iter_entry_points(group=group):
-                print(f"{group}\t{plugin}\t{plugin.dist}")
+            for plugin in importlib_metadata.entry_points(group=group):
+                plugin_info = f"{plugin.dist.name} {plugin.dist.version}"
+                print(f"{group}\t{plugin.name} = {plugin.value}\t{plugin_info}")
         sys.exit(0)
 
 
@@ -1125,11 +1122,7 @@ def runner_input_json_file(available_inputs, namespace, input_file, downloadable
         elif input_file == "-":
             input_json = sys.stdin.read()
         else:
-            input_json = (
-                asyncio.get_event_loop()
-                .run_until_complete(make_read_source(False)(input_file, [], None))
-                .source_text
-            )
+            input_json = asyncio.run(make_read_source(False)(input_file, [], None)).source_text
         input_json = YAML(typ="safe", pure=True).load(input_json)
         if not isinstance(input_json, dict):
             raise Error.InputError("check JSON input; expected top-level object")
@@ -1756,7 +1749,7 @@ def configure(cfg=None, show=False, force=False, **kwargs):
         die("`miniwdl configure` is for interactive use")
 
     from datetime import datetime
-    import bullet  # type: ignore
+    import questionary  # type: ignore
     from xdg import XDG_CONFIG_HOME
 
     miniwdl_version = pkg_version()
@@ -1774,11 +1767,12 @@ def configure(cfg=None, show=False, force=False, **kwargs):
             cfg = os.path.join(XDG_CONFIG_HOME, "miniwdl.cfg")
 
         def yes(prompt):
-            return bullet.Bullet(prompt=prompt, choices=["No", "Yes"]).launch() == "Yes"
+            answer = questionary.confirm(prompt, qmark="").unsafe_ask()
+            return bool(answer)
 
         if os.path.exists(cfg):
             assert force
-            logger.warn("Proceeding will overwrite existing configuration file at " + cfg)
+            logger.warning("Proceeding will overwrite existing configuration file at " + cfg)
             sys.stderr.flush()
             if not yes("OVERWRITE?"):
                 sys.exit(0)
@@ -1803,9 +1797,9 @@ def yes(prompt):
                 print("\nCall cache JSON file storage directory: ~/.cache/miniwdl/")
 
                 if yes("OVERRIDE?"):
-                    options["call_cache"]["dir"] = bullet.Input(
-                        prompt="Call cache directory: ", strip=True
-                    ).launch()
+                    options["call_cache"]["dir"] = (
+                        questionary.text("Call cache directory: ").unsafe_ask().strip()
+                    )
 
             print(
                 textwrap.dedent(
@@ -1822,9 +1816,10 @@ def yes(prompt):
                 print("\nDownload cache directory: /tmp/miniwdl_download_cache")
 
                 if yes("OVERRIDE?"):
-                    options["download_cache"]["dir"] = bullet.Input(
-                        prompt="Download cache directory: ", strip=True
-                    ).launch()
+                    # use questionary text prompt for directory input
+                    options["download_cache"]["dir"] = (
+                        questionary.text("Download cache directory: ").unsafe_ask().strip()
+                    )
 
             print()
             if yes("Configure non-public Amazon s3:// access?"):
@@ -1854,7 +1849,8 @@ def yes(prompt):
         print(cfg_content)
         print()
         sys.stdout.flush()
-        os.makedirs(os.path.dirname(cfg), exist_ok=True)
+        if dn := os.path.dirname(cfg):
+            os.makedirs(dn, exist_ok=True)
         with open(cfg, "w") as outfile:
             print(
                 f"# miniwdl configure {miniwdl_version or '(version unknown)'} {datetime.utcnow()}Z",

WDL/Expr.py

@@ -786,7 +786,7 @@ class Struct(Base):
     """
     :type: Optional[str]
 
-    In WDL 2.0+ each struct literal may specify the intended struct type name.
+    In WDL 1.1+ each struct literal may specify the intended struct type name.
     """
 
     def __init__(
@@ -818,7 +818,7 @@ def children(self) -> Iterable[SourceNode]:
     def _infer_type(self, type_env: Env.Bindings[Type.Base]) -> Type.Base:
         object_type = Type.Object({k: v.type for k, v in self.members.items()})
         if not self.struct_type_name:
-            # pre-WDL 2.0: object literal with deferred typechecking
+            # pre-WDL 1.1: object literal with deferred typechecking
             return object_type
 
         # resolve struct type
@@ -839,6 +839,16 @@ def _infer_type(self, type_env: Env.Bindings[Type.Base]) -> Type.Base:
                 self, struct_type, object_type, exn.args[0] if exn.args else ""
             )
 
+        # but object_type.check() allows extra members in WDL 1.2
+        for member in self.members:
+            if member not in struct_type.members:
+                raise Error.StaticTypeMismatch(
+                    self,
+                    struct_type,
+                    object_type,
+                    f"no such member '{member}' in struct {self.struct_type_name}",
+                )
+
         return struct_type
 
     def _eval(self, env: Env.Bindings[Value.Base], stdlib: StdLib.Base) -> Value.Base:

WDL/Tree.py

@@ -266,6 +266,10 @@ class Task(SourceNode):
     """:type: Dict[str,WDL.Expr.Base]
 
     ``runtime{}`` section, with keys and corresponding expressions to be evaluated"""
+    requirements: Dict[str, Expr.Base]
+    """:type: Dict[str,WDL.Expr.Base]
+
+    ``requirements{}`` section (for WDL 1.2+ tasks; refers to same dict as ``runtime``)"""
     meta: Dict[str, Any]
     """:type: Dict[str,Any]
 
@@ -297,6 +301,7 @@ def __init__(
         self.outputs = outputs
         self.parameter_meta = parameter_meta
         self.runtime = runtime
+        self.requirements = self.runtime
         self.meta = meta
         self.effective_wdl_version = "1.0"  # overridden by Document.__init__
         # TODO: enforce validity constraints on parameter_meta and runtime
@@ -1588,7 +1593,7 @@ def _load(
     import_max_depth: int = 10,
     importer: Optional[Document] = None,
 ) -> Document:
-    return asyncio.get_event_loop().run_until_complete(
+    return asyncio.run(
         _load_async(
             uri,
             path=path,

WDL/Value.py

@@ -561,11 +561,19 @@ def from_json(type: Type.Base, value: Any) -> Base:
         return String(value)
     if isinstance(type, Type.Array) and isinstance(value, list):
         return Array(type.item_type, [from_json(type.item_type, item) for item in value])
-    if isinstance(type, Type.Pair) and isinstance(value, dict) and set(value) == {"left", "right"}:
+    if (
+        isinstance(type, Type.Pair)
+        and isinstance(value, dict)
+        and set(v.lower() for v in value.keys()) == {"left", "right"}
+    ):
+        lowercased_value = {k.lower(): v for k, v in value.items()}
         return Pair(
             type.left_type,
             type.right_type,
-            (from_json(type.left_type, value["left"]), from_json(type.right_type, value["right"])),
+            (
+                from_json(type.left_type, lowercased_value["left"]),
+                from_json(type.right_type, lowercased_value["right"]),
+            ),
         )
     if (
         isinstance(type, Type.Map)

WDL/_grammar.py

@@ -64,7 +64,7 @@
 ?task_section: task_input_decls
              | output_decls
              | meta_section
-             | runtime_section
+             | requirements_section
              | hints_section
              | task_env_decl -> noninput_decl
 
@@ -87,8 +87,8 @@
            | "[" [meta_value ("," meta_value)*] ","? "]" -> meta_array
 !meta_section: ("meta" | "parameter_meta") meta_object
 
-// task runtime section (key-expression pairs)
-runtime_section: ("runtime" | "requirements") "{" [runtime_kv (","? runtime_kv)*] "}"
+// task requirements section (key-expression pairs); some mixing with vestigial (pre-1.2) "runtime" terminology
+requirements_section: ("requirements" | "runtime") "{" [runtime_kv (","? runtime_kv)*] "}"
 runtime_kv: CNAME ":" expr
 
 // hints section

WDL/_parser.py

@@ -396,6 +396,17 @@ def runtime_section(self, meta, items):
             d[k] = v
         return {"runtime": d}
 
+    def requirements_section(self, meta, items):
+        d = dict()
+        assert all(isinstance(item, tuple) and len(item) == 2 for item in items)
+        for k, v in items:
+            if k in d:
+                raise Error.MultipleDefinitions(
+                    self._sp(meta), "duplicate keys in requirements section"
+                )
+            d[k] = v
+        return {"runtime": d}
+
     def hints_section(self, meta, items):
         return {"hints": items[0]}
 
@@ -407,8 +418,10 @@ def task(self, meta, items):
                     if k == "noninput_decl":
                         d["noninput_decls"].append(v)
                     elif k in d:
+                        if k == "runtime":
+                            k = "requirements/runtime"
                         raise Error.MultipleDefinitions(
-                            self._sp(meta), "redundant sections in task"
+                            self._sp(meta), f"redundant {k} sections in task"
                         )
                     else:
                         d[k] = v

WDL/runtime/config_templates/default.cfg

@@ -209,7 +209,7 @@ flock = true
 
 [download_aria2c]
 # see: https://github.com/chanzuckerberg/miniwdl/tree/main/tools_image
-docker = ghcr.io/miniwdl-ext/miniwdl-tools:Id_sha256_93faaef93f05d4bc101d45f3218392fde55701d3ff2ad38967a0a62e408749ec
+docker = ghcr.io/miniwdl-ext/miniwdl-tools:Id_sha256_d18297a683804ebc63cfd662ca0362b5a323be33c834272bb9b71dd438c44bc8
 
 
 [download_awscli]
@@ -222,14 +222,14 @@ docker = ghcr.io/miniwdl-ext/miniwdl-tools:Id_sha256_93faaef93f05d4bc101d45f3218
 # Failing all of the above, public S3 URIs can always be used.
 host_credentials = false
 # see: https://github.com/chanzuckerberg/miniwdl/tree/main/tools_image
-docker = ghcr.io/miniwdl-ext/miniwdl-tools:Id_sha256_93faaef93f05d4bc101d45f3218392fde55701d3ff2ad38967a0a62e408749ec
+docker = ghcr.io/miniwdl-ext/miniwdl-tools:Id_sha256_d18297a683804ebc63cfd662ca0362b5a323be33c834272bb9b71dd438c44bc8
 
 
 [download_gsutil]
 # current version:
 #   docker pull gcr.io/google.com/cloudsdktool/cloud-sdk:slim
 #   docker run gcr.io/google.com/cloudsdktool/cloud-sdk:slim gcloud version
-docker = gcr.io/google.com/cloudsdktool/cloud-sdk:517.0.0-slim
+docker = gcr.io/google.com/cloudsdktool/cloud-sdk:543.0.0-slim
 
 
 [call_cache]