Merge branch 'main' into mlin-wdl2-runtime

Author: mlin

.github/copilot-instructions.md

@@ -0,0 +1,11 @@
+You're working on the source code for `miniwdl`, the Workflow Description Language (WDL) runner and developer toolkit.
+
+See CONTRIBUTING.md for an overview of the codebase and development workflow. Note in particular that Python code should be linted with `mypy`, `ruff check`, and `ruff format`.
+
+For many tasks it'll be useful to refer to the WDL 1.2 specification, which you can find under spec/wdl-1.2/SPEC.md. The version changelog is spec/wdl-1.2/CHANGELOG.md, and the older version 1.1 spec is spec/wdl-1.1/SPEC.md.
+
+These development tutorials under docs/ introduce a few common ways the codebase is used and extended.
+- trace_identifiers.md -- basic syntax tree traversal
+- wdlviz.md -- generating graphviz diagrams from WDL source code
+- add_functions.md -- adding new functions to the standard library
+- assert.md -- adding a new WDL language feature, with parsing, type-checking, and runtime execution

.gitmodules

@@ -61,3 +61,14 @@
 [submodule "test_corpi/broadinstitute/warp"]
 	path = test_corpi/broadinstitute/warp
 	url = https://github.com/broadinstitute/warp.git
+[submodule "wdl-1.2"]
+	path = spec/wdl-1.2
+	url = https://github.com/openwdl/wdl.git
+	branch = wdl-1.2
+[submodule "wdl-1.1"]
+	path = spec/wdl-1.1
+	url = https://github.com/openwdl/wdl.git
+	branch = wdl-1.1
+[submodule "spec/wdl-tests"]
+	path = spec/wdl-tests
+	url = https://github.com/openwdl/wdl-tests.git

CONTRIBUTING.md

@@ -21,7 +21,7 @@ Generally, `python3 -m WDL ...` invokes the equivalent of the `miniwdl ...` entr
 The Makefile has a few typical flows:
 
 - `make` or `make test` runs the full test suite with code coverage report (takes several minutes)
-- `make qtest` runs most of the tests more quickly (by omitting some slower cases, and not tracking coverage)
+- `make qtest` runs most of the tests more quickly (by omitting some slower cases, not tracking coverage, and failing fast)
 - `make pretty` reformats the code with `ruff format`
 - `make check` validates the code with `ruff check` and `mypy`
 
@@ -117,5 +117,6 @@ The miniwdl test suite is located in the `tests/` directory and is organized int
     - true unit tests of isolated components
     - artificial WDL sources exercising various aspects of the runtime
     - tests of the parser and linter on a collection of WDL from external sources found under `test_corpi/`
+*   **WDL Spec Examples (spec_tests/):** Tests all the examples embedded in the WDL SPEC.md (extracted following the [WDL Markdown Test Specification](https://github.com/openwdl/wdl-tests/blob/main/docs/MarkdownTests.md)). The `config.yaml` file marks several examples xfail or skip, for various reasons commented there.
 *   **Bash-TAP Tests (\*.t):**  These files are shell scripts that use the `bash-tap` framework to run commands and check their output.  They primarily exercise the `miniwdl` command-line interface (parsing arguments, executing workflows/tasks, etc.) and associated shell integration.
 *   **Applied Tests (applied/):** These are bash-tap that run `miniwdl run` on complete WDL workflows, drawn from various sources, with specific input JSON files.  These tests are used to ensure that miniwdl correctly executes realistic workflows.

Dockerfile

@@ -9,9 +9,9 @@
 # or append 'bash' to that to enter interactive shell
 
 # start with ubuntu:20.04 plus some apt packages
-FROM ubuntu:20.04 as deps
-ENV LC_ALL C.UTF-8
-ENV LANG C.UTF-8
+FROM ubuntu:20.04 AS deps
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \
     python3-pip python3-setuptools tzdata wget zip git-core default-jre jq shellcheck docker.io
 RUN pip3 install yq
@@ -25,10 +25,11 @@ RUN mkdir miniwdl
 # https://github.com/actions/checkout/issues/760
 RUN git config --global --add safe.directory /home/wdler/miniwdl
 
-ENV PATH $PATH:/home/wdler/.local/bin
+ENV PATH=$PATH:/home/wdler/.local/bin
+ENV PYTHONPATH=/home/wdler/miniwdl:$PYTHONPATH
 COPY pyproject.toml /home/wdler/miniwdl
 RUN tomlq -r '(.project.dependencies + .project["optional-dependencies"].dev)[]' miniwdl/pyproject.toml \
     | xargs pip3 install --user && rm miniwdl/pyproject.toml
 
 # expectation -- mount miniwdl source tree at /home/wdler/miniwdl
-CMD make -C miniwdl unit_tests
+CMD ["make","-C","miniwdl","unit_tests"]

Makefile

@@ -6,15 +6,17 @@ test: check_check check unit_tests integration_tests
 # fail fast
 qtest:
 	python3 tests/no_docker_services.py
-	pytest -vx --tb=short -n `python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'` --dist=loadscope tests
+	pytest -vx --tb=short -n auto --dist=loadscope tests
+	pytest -vx --tb=short -n auto tests/spec_tests/spec_tests.py
 	python3 -m unittest tests.test_cli_argcomplete
 	prove -v tests/{check,eval,runner,zip}.t
 	python3 tests/no_docker_services.py
 
 unit_tests:
-	pytest -v --tb=short -n `python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'` --dist=loadscope --cov=WDL tests
+	pytest -v --tb=short -n auto --dist=loadscope --cov=WDL tests
+	pytest -v --tb=short -n auto tests/spec_tests/spec_tests.py
 	python3 -m unittest tests.test_cli_argcomplete
-	python3 tests/no_docker_services.py
+	python3 tests/no_docker_services.py	
 
 integration_tests:
 	prove -v tests/{check,eval,runner,zip,multi_line_strings}.t
@@ -79,4 +81,10 @@ doc:
 
 docs: doc
 
-.PHONY: check check_check pretty test qtest docker doc docs pypi_test pypi bdist ci_housekeeping unit_tests integration_tests skylab_bulk_rna DVGLx viral_assemble
+.PHONY: \
+  pretty check check_check \
+  test qtest unit_tests integration_tests \
+  skylab_bulk_rna DVGLx viral_assemble viral_refbased singularity_tests \
+  doc docs \
+  bdist pypi_test pypi \
+  ci_housekeeping ci_unit_tests

WDL/StdLib.py

@@ -653,8 +653,7 @@ def _call_eager(self, expr: "Expr.Apply", arguments: List[Value.Base]) -> Value.
 
 
 class _ComparisonOperator(EagerFunction):
-    # < > <= >= operators; reuses some of the equatability logic, but not valid for optional or
-    # compound types.
+    # < > <= >= operators
 
     name: str
     op: Callable
@@ -665,7 +664,9 @@ def __init__(self, name: str, op: Callable) -> None:
 
     def infer_type(self, expr: "Expr.Apply") -> Type.Base:
         assert len(expr.arguments) == 2
-        if not expr.arguments[0].type.comparable(expr.arguments[1].type):
+        if not expr.arguments[0].type.comparable(
+            expr.arguments[1].type, check_quant=expr._check_quant
+        ):
             raise Error.IncompatibleOperand(
                 expr,
                 "Cannot compare {} and {}".format(

WDL/Type.py

@@ -132,26 +132,29 @@ def __str__(self) -> str:
     def __eq__(self, rhs: typing.Any) -> bool:
         return isinstance(rhs, Base) and str(self) == str(rhs)
 
-    def equatable(self, rhs: "Base", compound: bool = False) -> bool:
+    def equatable(self, rhs: "Base", *, compound: bool = False) -> bool:
         """
         Check if values of the given types may be equated with the WDL == operator (and its
         negation). This mostly requires they have the same type, with quirks like ignoring
         quantifiers and Int/Float coercion (allowed at 'top level' but not within compound types).
         """
         return isinstance(rhs, (type(self), Any))  # intentionally ignores optional quantifier
 
-    def comparable(self, rhs: "Base") -> bool:
+    def comparable(self, rhs: "Base", *, check_quant: bool = True) -> bool:
         """
         Check if values of the given types may be compared with the WDL comparison operators (<, <=,
-        >, >=). Currently these are allowed only for non-optional primitive values.
+        >, >=).
         """
+        # Since this is currently only allowed for primitive types, we can code all cases here
+        # instead of fanning out to subclasses (still possible if we need to in the future).
         allowed = (Int, Float, String, Boolean)
-        return (
-            isinstance(self, allowed)
-            and not self.optional
-            and isinstance(rhs, allowed)
-            and not rhs.optional
-        )
+        if not (isinstance(self, allowed) and isinstance(rhs, allowed)):
+            return False
+        if check_quant and (self.optional or rhs.optional):
+            return False
+        if isinstance(self, (Int, Float)) and isinstance(rhs, (Int, Float)):
+            return True
+        return isinstance(rhs, type(self))
 
 
 class Any(Base):
@@ -172,7 +175,7 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
     def __str__(self) -> str:
         return "None" if self._optional else "Any"
 
-    def equatable(self, rhs, compound: bool = False):
+    def equatable(self, rhs, *, compound: bool = False):
         return True
 
 
@@ -197,9 +200,9 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
             return self._check_optional(rhs, check_quant)
         super().check(rhs, check_quant)
 
-    def equatable(self, rhs, compound: bool = False):
+    def equatable(self, rhs, *, compound: bool = False):
         # per WDL spec, Int/Float can be equated directly, but not as part of a compound type
-        return super().equatable(rhs, compound) or (not compound and isinstance(rhs, Int))
+        return super().equatable(rhs, compound=compound) or (not compound and isinstance(rhs, Int))
 
 
 class Int(Base):
@@ -214,8 +217,10 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
             return self._check_optional(rhs, check_quant)
         super().check(rhs, check_quant)
 
-    def equatable(self, rhs, compound: bool = False):
-        return super().equatable(rhs, compound) or (not compound and isinstance(rhs, Float))
+    def equatable(self, rhs, *, compound: bool = False):
+        return super().equatable(rhs, compound=compound) or (
+            not compound and isinstance(rhs, Float)
+        )
 
 
 class File(Base):
@@ -311,7 +316,7 @@ def copy(self, optional: Optional[bool] = None, nonempty: Optional[bool] = None)
             setattr(ans, "_nonempty", nonempty)
         return ans
 
-    def equatable(self, rhs, compound: bool = False):
+    def equatable(self, rhs, *, compound: bool = False):
         # intentionally ignores optional and nonempty quantifiers:
         return isinstance(rhs, Array) and self.item_type.equatable(rhs.item_type, compound=True)
 
@@ -388,7 +393,7 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
             return
         super().check(rhs, check_quant)
 
-    def equatable(self, rhs, compound: bool = False):
+    def equatable(self, rhs, *, compound: bool = False):
         return (
             isinstance(rhs, Map)
             and self.item_type[0].equatable(rhs.item_type[0], compound=True)
@@ -436,7 +441,7 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
             return self._check_optional(rhs, check_quant)
         super().check(rhs, check_quant)
 
-    def equatable(self, rhs, compound=True):
+    def equatable(self, rhs, *, compound: bool = True):
         return (
             isinstance(rhs, Pair)
             and self.left_type.equatable(rhs.left_type, compound=True)
@@ -501,7 +506,7 @@ def parameters(self) -> Iterable[Base]:
         assert self.members is not None
         return self.members.values()
 
-    def equatable(self, rhs, coerce=True):
+    def equatable(self, rhs, *, compound: bool = False):
         return isinstance(rhs, StructInstance) and self.type_id == rhs.type_id
 
 
@@ -558,7 +563,7 @@ def check(self, rhs: Base, check_quant: bool = True) -> None:
             return
         raise TypeError()
 
-    def equatable(self, rhs, compound: bool = False):
+    def equatable(self, rhs, *, compound: bool = False):
         return False