Replace usage of spirv-* binaries with spirv-tools rust crate

.github/workflows/ci.yaml

@@ -11,25 +11,40 @@ jobs:
     name: Test
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os: [macOS-latest, ubuntu-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
+          - os: macOS-latest
+            target: x86_64-apple-darwin
     runs-on: ${{ matrix.os }}
     env:
       spirv_tools_version: "20200928"
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
+        with:
+          submodules: "false"
       # Ubuntu does have `brew install spirv-tools`, but it installs from
       # source and so takes >8 minutes.
       - if: ${{ runner.os == 'Linux' }}
+        name: Linux - Install native dependencies
         run: |
           sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
       - if: ${{ runner.os == 'macOS' }}
+        name: Mac - Install spirv-tools
         run: brew install spirv-tools
       # Currently SPIR-V tools aren't available in any package manager
       # on Windows that put the tools in the path.
       - if: ${{ runner.os == 'Windows' }}
+        name: Windows - Install spirv-tools
         shell: bash
         run: |
           tmparch=$(mktemp)
@@ -39,25 +54,42 @@ jobs:
       - if: ${{ runner.os == 'Windows' }}
         # Runs separately to add spir-v tools to Powershell's Path.
         run: echo "$HOME/spirv-tools/install/bin" >> $env:GITHUB_PATH
-      - run: rustup component add rust-src rustc-dev llvm-tools-preview
-      # See: https://github.com/EmbarkStudios/rust-gpu/issues/84
-      - if: ${{ runner.os == 'macOS' }}
-        run: cargo test --workspace --exclude example-runner
-      - if: ${{ runner.os != 'macOS' }}
-        run: cargo test --workspace
+      - name: Install rustup components
+        run: rustup component add rust-src rustc-dev llvm-tools-preview
+      # Fetch dependencies in a separate step to clearly show how long each part
+      # of the testing takes
+      - name: cargo fetch
+        run: cargo fetch --target ${{ matrix.target }}
+      - name: Run tests
+        shell: bash
+        run: .github/workflows/test.sh
+
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
-      - run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
-      - run: |
+        with:
+          submodules: "false"
+      - name: Install native dependencies
+        run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
+      - name: Install spirv-tools
+        run: |
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
-      - run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
-      - run: cargo fmt --all -- --check
-      - run: cargo clippy --workspace --all-targets -- -D warnings
+      - name: Install rustup components
+        run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
+      - name: Rustfmt
+        run: cargo fmt --all -- --check
+      - name: cargo fetch
+        run: cargo fetch
+      - name: Clippy
+        run: .github/workflows/clippy.sh
+
   cargo-deny:
     runs-on: ubuntu-latest
     steps:

.github/workflows/clippy.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+function clippy() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+function clippy_no_features() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+# Core crates
+clippy spirv-tools-sys
+clippy spirv-tools
+clippy rustc_codegen_spirv
+clippy spirv-builder
+
+# Examples
+clippy examples/example-runner
+clippy examples/wgpu-example-runner
+
+clippy_no_features examples/example-runner-cpu
+clippy_no_features examples/example-shader
+clippy_no_features examples/wgpu-example-shader

.github/workflows/test.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+os=$1
+
+function cargo_test() {
+    echo ::group::"$1 build"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT"
+    echo ::endgroup::
+}
+
+function cargo_test_no_features() {
+    echo ::group::"$1 build"
+    cargo test --manifest-path "$1/Cargo.toml" --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test --manifest-path "$1/Cargo.toml"
+    echo ::endgroup::
+}
+
+# Core crates
+cargo_test spirv-tools-sys
+cargo_test spirv-tools
+cargo_test rustc_codegen_spirv
+cargo_test spirv-builder
+
+# Examples
+# See: https://github.com/EmbarkStudios/rust-gpu/issues/84
+if [[ -z "${CI}" && "$os" != "macOS" ]]; then
+    cargo_test examples/example-runner
+fi
+
+cargo_test examples/wgpu-example-runner
+
+cargo_test_no_features examples/example-runner-cpu
+cargo_test_no_features examples/example-shader
+cargo_test_no_features examples/wgpu-example-shader

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "spirv-tools-sys/spirv-headers"]
+	path = spirv-tools-sys/spirv-headers
+	url = https://github.com/KhronosGroup/SPIRV-Headers.git
+[submodule "spirv-tools-sys/spirv-tools"]
+	path = spirv-tools-sys/spirv-tools
+	url = https://github.com/EmbarkStudios/SPIRV-Tools.git
+	branch = patch-to-string

Cargo.toml

@@ -5,7 +5,10 @@ members = [
     "examples/wgpu-example-runner",
     "examples/example-shader",
     "examples/wgpu-example-shader",
+
     "rustc_codegen_spirv",
     "spirv-builder",
     "spirv-std",
+    "spirv-tools",
+    "spirv-tools-sys",
 ]

docs/src/introduction.md

@@ -1,26 +1,33 @@
 # Introduction
+
 Welcome to the Rust-GPU dev guide! This documentation is meant for documenting
 how to use and develop on Rust-GPU.
 
 ## Getting started
 
 1. Clone the repository.
 
+    ```shell
+    git clone --recurse-submodules https://github.com/EmbarkStudios.com/rust-gpu
+    ```
+
 1. Install the prerequisites using the provided setup script. From the root of the project, run:
 
     MacOS, Linux:
+
     ```shell
     sh setup.sh
     ```
 
     Windows:
-    ```
+
+    ```shell
     setup.bat
     ```
 
     The setup script installs nightly Rust (required for now, see [#78](https://github.com/EmbarkStudios/rust-gpu/issues/78) for tracking issue).
 
-1. Install [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) and add it to your PATH (for now, eventually we will automatically build and link it instead of calling executables)
+1. **optional** Install [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) and add it to your `PATH`. You can skip this step if you just want to run examples with the defaults. See [Using installed SPIRV-Tools](#using-installed-spirv-tools) if you decide to go with this option.
 
 1. Next, look at the [examples](examples) folder. There are two projects here: [examples/example-shader](examples/example-shader) and [examples/example-runner](examples/example-runner). The example-shader project is a "GPU crate", one that will be compiled to a SPIR-V module. The example-runner project is a normal, CPU crate that uses vulkan to consume the example-shader SPIR-V module to display a shader.
 
@@ -36,15 +43,13 @@ how to use and develop on Rust-GPU.
 
 Be aware that this project is in a very early phase - if the above doesn't work, please [file an issue](https://github.com/EmbarkStudios/rust-gpu/issues)!
 
-## Getting started, for power users who don't want to use spirv-builder.
+## Getting started, for power users who don't want to use spirv-builder
 
 If you would like to build the compiler, `rustc_codegen_spirv` is the relevant folder. Install the prerequisites, as above, then, `cd rustc_codegen_spirv && cargo build`. This produces an .so file, located at `./target/debug/librustc_codegen_spirv.so` (or `.dll`/`.dylib` depending on your platform).
 
 This file is a dynamically loaded backend for rustc - you may tell rustc to use it as a backend through the `-Z codegen-backend=...` flag. To pass this to rustc through cargo, set the environment variable `RUSTFLAGS="-Z codegen-backend=$PATH_TO_FILE"`.
 
-Then, when building a GPU crate, we need to configure some flags when we call cargo. First, we need to build libcore
-ourselves - we obviously have no SPIR-V libcore installed on our system! Use the flag `-Z build-std=core`. Then, we need
-to tell rustc to generate SPIR-V instead of x86 code: `--target spirv-unknown-unknown`.
+Then, when building a GPU crate, we need to configure some flags when we call cargo. First, we need to build libcore ourselves - we obviously have no SPIR-V libcore installed on our system! Use the flag `-Z build-std=core`. Then, we need to tell rustc to generate SPIR-V instead of x86 code: `--target spirv-unknown-unknown`.
 
 Overall, building your own SPIR-V crate looks like:
 
@@ -61,3 +66,15 @@ To create a GPU crate, look at the [examples/example-shader](examples/example-sh
 
 This is all a little convoluted, hence the [spirv-builder](spirv-builder) crate handles a lot of this.
 
+## Using installed SPIRV-Tools
+
+By default, all of the crates and examples in this repo will compile the `spirv-tools-sys` crate, including a lot of C++ code from [SPIRV-Tools](https://github.com/EmbarkStudios/SPIRV-Tools). If you don't want to build the C++ code because you already have [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) installed, or just don't want to spend more time compiling, you can build/run the crate with the `use-installed-tools` feature.
+
+```shell
+cargo run \
+    --manifest-path examples/example-runner/Cargo.toml \
+    --features use-installed-tools \
+    --no-default-features
+```
+
+You should see `warning: use-installed-tools feature on, skipping compilation of C++ code` during the compilation, but otherwise the build will function just the same as if you compiled the C++ code, with the exception that it will fail if you don't have SPIRV-Tools installed correctly.

examples/example-runner-cpu/Cargo.toml

@@ -14,4 +14,3 @@ spirv-std = { path = "../../spirv-std" }
 
 # for parallelism, not really needed though
 rayon = "1.5"
-

examples/example-runner/Cargo.toml

@@ -5,6 +5,12 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 ash = "0.31"
 ash-window = "0.5"
@@ -16,4 +22,4 @@ winit = "0.23.0"
 ash-molten = { git = "https://github.com/EmbarkStudios/ash-molten", branch = "moltenvk-1.1.0" }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }

examples/wgpu-example-runner/Cargo.toml

@@ -5,13 +5,19 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 wgpu = "0.6.0"
 futures = { version = "0.3", default-features = false, features = ["std", "executor"] }
 winit = { version = "0.22.1", features = ["web-sys"] }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 wgpu-subscriber = "0.1.0"

rustc_codegen_spirv/Cargo.toml

@@ -12,13 +12,32 @@ repository = "https://github.com/EmbarkStudios/rust-gpu"
 [lib]
 crate-type = ["dylib"]
 
+[features]
+# By default, the use-compiled-tools is enabled, as doesn't require additional
+# setup steps for the user. This does however mean that you will need to disable
+# default features and explicitly enable `use-installed-tools` if you are using
+# this in an environment with spirv-tools in PATH, and you don't want to take
+# the compile time cost
+default = ["use-compiled-tools"]
+# If enabled, uses spirv-tools binaries installed in PATH, instead of
+# compiling and linking the spirv-tools C++ code
+use-installed-tools = ["spirv-tools/use-installed-tools"]
+# If enabled will compile and link the C++ code for the spirv tools, the compiled
+# version is preferred if both this and `use-installed-tools` are enabled
+use-compiled-tools = ["spirv-tools/use-compiled-tools"]
+
 [dependencies]
 bimap = "0.5"
 rspirv = { git = "https://github.com/gfx-rs/rspirv.git", rev = "f11f8797bd4df2d1d22cf10767b39a5119c57551" }
 tar = "0.4.30"
 thiserror = "1.0.20"
 topological-sort = "0.1"
 
+[dependencies.spirv-tools]
+version = "0.1.0"
+path = "../spirv-tools"
+default-features = false
+
 [dev-dependencies]
 pretty_assertions = "0.6"
 tempfile = "3.1"