Replace usage of spirv-* binaries with spirv-tools rust crate

.github/workflows/ci.yaml

@@ -16,20 +16,28 @@ jobs:
     env:
       spirv_tools_version: "20200928"
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
+        with:
+          submodules: "false"
       # Ubuntu does have `brew install spirv-tools`, but it installs from
       # source and so takes >8 minutes.
       - if: ${{ runner.os == 'Linux' }}
+        name: Linux - Install native dependencies
         run: |
           sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
       - if: ${{ runner.os == 'macOS' }}
+        name: Mac - Install spirv-tools
         run: brew install spirv-tools
       # Currently SPIR-V tools aren't available in any package manager
       # on Windows that put the tools in the path.
       - if: ${{ runner.os == 'Windows' }}
+        name: Windows - Install spirv-tools
         shell: bash
         run: |
           tmparch=$(mktemp)
@@ -39,25 +47,41 @@ jobs:
       - if: ${{ runner.os == 'Windows' }}
         # Runs separately to add spir-v tools to Powershell's Path.
         run: echo "$HOME/spirv-tools/install/bin" >> $env:GITHUB_PATH
-      - run: rustup component add rust-src rustc-dev llvm-tools-preview
-      # See: https://github.com/EmbarkStudios/rust-gpu/issues/84
-      - if: ${{ runner.os == 'macOS' }}
-        run: cargo test --workspace --exclude example-runner
-      - if: ${{ runner.os != 'macOS' }}
-        run: cargo test --workspace
+      - name: Install rustup components
+        run: rustup component add rust-src rustc-dev llvm-tools-preview
+      # Fetch dependencies in a separate step to clearly show how long each part
+      # of the testing takes
+      - name: cargo fetch
+        run: cargo fetch
+      - name: Run tests
+        run: .github/workflows/test.sh
+
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
-      - run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
-      - run: |
+        with:
+          submodules: "false"
+      - name: Install native dependencies
+        run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
+      - name: Install spirv-tools
+        run: |
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
-      - run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
-      - run: cargo fmt --all -- --check
-      - run: cargo clippy --workspace --all-targets -- -D warnings
+      - name: Install rustup components
+        run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
+      - name: Rustfmt
+        run: cargo fmt --all -- --check
+      - name: cargo fetch
+        run: cargo fetch
+      - name: Clippy
+        run: .github/workflows/clippy.sh
+
   cargo-deny:
     runs-on: ubuntu-latest
     steps:

.github/workflows/clippy.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+function clippy() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+function clippy_no_features() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+# Core crates
+clippy spirv-tools-sys
+clippy spirv-tools
+clippy rustc_codegen_spirv
+clippy spirv-builder
+
+# Examples
+clippy examples/example-runner
+clippy examples/wgpu-example-runner
+
+clippy_no_features examples/example-runner-cpu
+clippy_no_features examples/example-shader
+clippy_no_features examples/wgpu-example-shader

.github/workflows/test.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+os=$1
+
+function cargo_test() {
+    echo ::group::"$1 build"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT"
+    echo ::endgroup::
+}
+
+function cargo_test_no_features() {
+    echo ::group::"$1 build"
+    cargo test --manifest-path "$1/Cargo.toml" --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test --manifest-path "$1/Cargo.toml"
+    echo ::endgroup::
+}
+
+# Core crates
+cargo_test spirv-tools-sys
+cargo_test spirv-tools
+cargo_test rustc_codegen_spirv
+cargo_test spirv-builder
+
+# Examples
+# See: https://github.com/EmbarkStudios/rust-gpu/issues/84
+if [[ -z "${CI}" && "$os" != "macOS" ]]; then
+    cargo_test examples/example-runner
+fi
+
+cargo_test examples/wgpu-example-runner
+
+cargo_test_no_features examples/example-runner-cpu
+cargo_test_no_features examples/example-shader
+cargo_test_no_features examples/wgpu-example-shader

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "spirv-tools-sys/spirv-headers"]
+	path = spirv-tools-sys/spirv-headers
+	url = https://github.com/KhronosGroup/SPIRV-Headers.git
+[submodule "spirv-tools-sys/spirv-tools"]
+	path = spirv-tools-sys/spirv-tools
+	url = https://github.com/EmbarkStudios/SPIRV-Tools.git
+	branch = patch-to-string

Cargo.toml

@@ -5,7 +5,10 @@ members = [
     "examples/wgpu-example-runner",
     "examples/example-shader",
     "examples/wgpu-example-shader",
+
     "rustc_codegen_spirv",
     "spirv-builder",
     "spirv-std",
+    "spirv-tools",
+    "spirv-tools-sys",
 ]

examples/example-runner-cpu/Cargo.toml

@@ -14,4 +14,3 @@ spirv-std = { path = "../../spirv-std" }
 
 # for parallelism, not really needed though
 rayon = "1.5"
-

examples/example-runner/Cargo.toml

@@ -5,6 +5,12 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 ash = "0.31"
 ash-window = "0.5"
@@ -16,4 +22,4 @@ winit = "0.23.0"
 ash-molten = { git = "https://github.com/EmbarkStudios/ash-molten", branch = "moltenvk-1.1.0" }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }

examples/wgpu-example-runner/Cargo.toml

@@ -5,13 +5,19 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 wgpu = "0.6.0"
 futures = { version = "0.3", default-features = false, features = ["std", "executor"] }
 winit = { version = "0.22.1", features = ["web-sys"] }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 wgpu-subscriber = "0.1.0"

rustc_codegen_spirv/Cargo.toml

@@ -12,13 +12,32 @@ repository = "https://github.com/EmbarkStudios/rust-gpu"
 [lib]
 crate-type = ["dylib"]
 
+[features]
+# By default, the use-compiled-tools is enabled, as doesn't require additional
+# setup steps for the user. This does however mean that you will need to disable
+# default features and explicitly enable `use-installed-tools` if you are using
+# this in an environment with spirv-tools in PATH, and you don't want to take
+# the compile time cost
+default = ["use-compiled-tools"]
+# If enabled, uses spirv-tools binaries installed in PATH, instead of
+# compiling and linking the spirv-tools C++ code
+use-installed-tools = ["spirv-tools/use-installed-tools"]
+# If enabled will compile and link the C++ code for the spirv tools, the compiled
+# version is preferred if both this and `use-installed-tools` are enabled
+use-compiled-tools = ["spirv-tools/use-compiled-tools"]
+
 [dependencies]
 bimap = "0.5"
 rspirv = { git = "https://github.com/gfx-rs/rspirv.git", rev = "f11f8797bd4df2d1d22cf10767b39a5119c57551" }
 tar = "0.4.30"
 thiserror = "1.0.20"
 topological-sort = "0.1"
 
+[dependencies.spirv-tools]
+version = "0.1.0"
+path = "../spirv-tools"
+default-features = false
+
 [dev-dependencies]
 pretty_assertions = "0.6"
 tempfile = "3.1"