Add spirv-tools and spirv-tools-sys crates to wrap usage of spirv-tools (#179)

* Add spirv-headers and spirv-tools as submodules

* Add simple generator and the generated code needed for compilation

* Add first pass on spirv-tools-sys

* Add first pass on spirv-tools

* Replace invocation of spirv-opt with spirv-tools crate

* Use C++11

* Placate clippy

* Add validation, replacing spirv-val with the spirv-tools crate

* Fix MSVC warning

* Use patched spirv-tools

* Fixup metadata

* Add same compiler flags as "official" build scripts

* Update spirv-tools and generated files

* Fixup

* Add assembler and example

* Use assembler in tests

* Oops, fix macos TARGET_OS

* write -> write_all

* Start splitting spirv-tools into a compiled vs tool feature set

* Checkpointing

* Checkpoint

* Boop

* Get tests to work both with installed and compiled tools

* Cleanup CI config

* Splits steps to clearly show how long each part of a longer (eg test)
step actually takes
* Label all steps

* Explicitly disable submodule checkout

* Rustfmt

* Rename features for consistency and fix clippy warnings

* Split "core" crates from examples

* Add run_clippy bash script

* Add test script

* Remove x flag

* Newline

* Actually print out errors from running val/opt

* Revert drive-by import merging

* Change intro to take the changes this PR has into account

* Actually run tests on Windows

* Fetch only the host target to reduce fetch times

* Add more info when a spirv tool returns a non-zero exit code

* Rustfmt

* Switch tool assembler to use files to see if it fixes windows

* Use files for input and output for now until I can figure out Windows being dumb

* Fix API docs generation

* Compile and use C++ code to check Windows issue

* Return to use installed tools

Author: Jake-Shadle

.github/workflows/ci.yaml

@@ -11,25 +11,40 @@ jobs:
     name: Test
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os: [macOS-latest, ubuntu-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
+          - os: macOS-latest
+            target: x86_64-apple-darwin
     runs-on: ${{ matrix.os }}
     env:
       spirv_tools_version: "20200928"
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
+        with:
+          submodules: "false"
       # Ubuntu does have `brew install spirv-tools`, but it installs from
       # source and so takes >8 minutes.
       - if: ${{ runner.os == 'Linux' }}
+        name: Linux - Install native dependencies
         run: |
           sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
       - if: ${{ runner.os == 'macOS' }}
+        name: Mac - Install spirv-tools
         run: brew install spirv-tools
       # Currently SPIR-V tools aren't available in any package manager
       # on Windows that put the tools in the path.
       - if: ${{ runner.os == 'Windows' }}
+        name: Windows - Install spirv-tools
         shell: bash
         run: |
           tmparch=$(mktemp)
@@ -39,25 +54,42 @@ jobs:
       - if: ${{ runner.os == 'Windows' }}
         # Runs separately to add spir-v tools to Powershell's Path.
         run: echo "$HOME/spirv-tools/install/bin" >> $env:GITHUB_PATH
-      - run: rustup component add rust-src rustc-dev llvm-tools-preview
-      # See: https://github.com/EmbarkStudios/rust-gpu/issues/84
-      - if: ${{ runner.os == 'macOS' }}
-        run: cargo test --workspace --exclude example-runner
-      - if: ${{ runner.os != 'macOS' }}
-        run: cargo test --workspace
+      - name: Install rustup components
+        run: rustup component add rust-src rustc-dev llvm-tools-preview
+      # Fetch dependencies in a separate step to clearly show how long each part
+      # of the testing takes
+      - name: cargo fetch
+        run: cargo fetch --target ${{ matrix.target }}
+      - name: Run tests
+        shell: bash
+        run: .github/workflows/test.sh
+
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
+      # Note that we are explicitly NOT checking out submodules, to validate
+      # that we haven't accidentally enabled spirv-tools native compilation
+      # and regressed CI times
       - uses: actions/checkout@v2
-      - run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
-      - run: |
+        with:
+          submodules: "false"
+      - name: Install native dependencies
+        run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
+      - name: Install spirv-tools
+        run: |
           mkdir "${HOME}/spirv-tools"
           curl -fL https://storage.googleapis.com/spirv-tools/artifacts/prod/graphics_shader_compiler/spirv-tools/linux-clang-release/continuous/1305/20201026-063148/install.tgz | tar -xz -C "${HOME}/spirv-tools"
           echo "${HOME}/spirv-tools/install/bin" >> $GITHUB_PATH
-      - run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
-      - run: cargo fmt --all -- --check
-      - run: cargo clippy --workspace --all-targets -- -D warnings
+      - name: Install rustup components
+        run: rustup component add rustfmt clippy rust-src rustc-dev llvm-tools-preview
+      - name: Rustfmt
+        run: cargo fmt --all -- --check
+      - name: cargo fetch
+        run: cargo fetch
+      - name: Clippy
+        run: .github/workflows/clippy.sh
+
   cargo-deny:
     runs-on: ubuntu-latest
     steps:

.github/workflows/clippy.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+function clippy() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+function clippy_no_features() {
+    echo ::group::"$1"
+    cargo clippy \
+        --manifest-path "$1/Cargo.toml" \
+        --all-targets \
+        -- -D warnings
+    echo ::endgroup::
+}
+
+# Core crates
+clippy spirv-tools-sys
+clippy spirv-tools
+clippy rustc_codegen_spirv
+clippy spirv-builder
+
+# Examples
+clippy examples/example-runner
+clippy examples/wgpu-example-runner
+
+clippy_no_features examples/example-runner-cpu
+clippy_no_features examples/example-shader
+clippy_no_features examples/wgpu-example-shader

.github/workflows/deploy_docs.yml

@@ -13,7 +13,7 @@ jobs:
       - run: brew install mdbook spirv-tools
       - run: mkdir docs-build/
       - run: $(cd docs && mdbook build -d ../docs-build/book)
-      - run: cargo doc
+      - run: .github/workflows/docs.sh
       - run: mv target/doc docs-build/api
       - uses: JamesIves/[email protected]
         with:

.github/workflows/docs.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+function doc() {
+    echo ::group::"$1"
+    cargo doc \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT"
+    echo ::endgroup::
+}
+
+# Core crates only!
+doc spirv-tools-sys
+doc spirv-tools
+doc rustc_codegen_spirv
+doc spirv-builder

.github/workflows/test.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${CI}" ]]; then
+    FEAT="use-compiled-tools"
+else
+    FEAT="use-installed-tools"
+fi
+
+os=$1
+
+function cargo_test() {
+    echo ::group::"$1 build"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT" \
+        --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test \
+        --manifest-path "$1/Cargo.toml" \
+        --no-default-features \
+        --features "$FEAT"
+    echo ::endgroup::
+}
+
+function cargo_test_no_features() {
+    echo ::group::"$1 build"
+    cargo test --manifest-path "$1/Cargo.toml" --no-run
+    echo ::endgroup::
+
+    echo ::group::"$1 test"
+    cargo test --manifest-path "$1/Cargo.toml"
+    echo ::endgroup::
+}
+
+# Core crates
+cargo_test spirv-tools-sys
+cargo_test spirv-tools
+cargo_test rustc_codegen_spirv
+cargo_test spirv-builder
+
+# Examples
+# See: https://github.com/EmbarkStudios/rust-gpu/issues/84
+if [[ -z "${CI}" && "$os" != "macOS" ]]; then
+    cargo_test examples/example-runner
+fi
+
+cargo_test examples/wgpu-example-runner
+
+cargo_test_no_features examples/example-runner-cpu
+cargo_test_no_features examples/example-shader
+cargo_test_no_features examples/wgpu-example-shader

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "spirv-tools-sys/spirv-headers"]
+	path = spirv-tools-sys/spirv-headers
+	url = https://github.com/KhronosGroup/SPIRV-Headers.git
+[submodule "spirv-tools-sys/spirv-tools"]
+	path = spirv-tools-sys/spirv-tools
+	url = https://github.com/EmbarkStudios/SPIRV-Tools.git
+	branch = patch-to-string

Cargo.lock

@@ -5,7 +5,10 @@ members = [
     "examples/wgpu-example-runner",
     "examples/example-shader",
     "examples/wgpu-example-shader",
+
     "rustc_codegen_spirv",
     "spirv-builder",
     "spirv-std",
+    "spirv-tools",
+    "spirv-tools-sys",
 ]

Cargo.toml

@@ -1,26 +1,33 @@
 # Introduction
+
 Welcome to the Rust-GPU dev guide! This documentation is meant for documenting
 how to use and develop on Rust-GPU.
 
 ## Getting started
 
 1. Clone the repository.
 
+    ```shell
+    git clone --recurse-submodules https://github.com/EmbarkStudios.com/rust-gpu
+    ```
+
 1. Install the prerequisites using the provided setup script. From the root of the project, run:
 
     MacOS, Linux:
+
     ```shell
     sh setup.sh
     ```
 
     Windows:
-    ```
+
+    ```shell
     setup.bat
     ```
 
     The setup script installs nightly Rust (required for now, see [#78](https://github.com/EmbarkStudios/rust-gpu/issues/78) for tracking issue).
 
-1. Install [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) and add it to your PATH (for now, eventually we will automatically build and link it instead of calling executables)
+1. **optional** Install [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) and add it to your `PATH`. You can skip this step if you just want to run examples with the defaults. See [Using installed SPIRV-Tools](#using-installed-spirv-tools) if you decide to go with this option.
 
 1. Next, look at the [examples](examples) folder. There are two projects here: [examples/example-shader](examples/example-shader) and [examples/example-runner](examples/example-runner). The example-shader project is a "GPU crate", one that will be compiled to a SPIR-V module. The example-runner project is a normal, CPU crate that uses vulkan to consume the example-shader SPIR-V module to display a shader.
 
@@ -36,15 +43,13 @@ how to use and develop on Rust-GPU.
 
 Be aware that this project is in a very early phase - if the above doesn't work, please [file an issue](https://github.com/EmbarkStudios/rust-gpu/issues)!
 
-## Getting started, for power users who don't want to use spirv-builder.
+## Getting started, for power users who don't want to use spirv-builder
 
 If you would like to build the compiler, `rustc_codegen_spirv` is the relevant folder. Install the prerequisites, as above, then, `cd rustc_codegen_spirv && cargo build`. This produces an .so file, located at `./target/debug/librustc_codegen_spirv.so` (or `.dll`/`.dylib` depending on your platform).
 
 This file is a dynamically loaded backend for rustc - you may tell rustc to use it as a backend through the `-Z codegen-backend=...` flag. To pass this to rustc through cargo, set the environment variable `RUSTFLAGS="-Z codegen-backend=$PATH_TO_FILE"`.
 
-Then, when building a GPU crate, we need to configure some flags when we call cargo. First, we need to build libcore
-ourselves - we obviously have no SPIR-V libcore installed on our system! Use the flag `-Z build-std=core`. Then, we need
-to tell rustc to generate SPIR-V instead of x86 code: `--target spirv-unknown-unknown`.
+Then, when building a GPU crate, we need to configure some flags when we call cargo. First, we need to build libcore ourselves - we obviously have no SPIR-V libcore installed on our system! Use the flag `-Z build-std=core`. Then, we need to tell rustc to generate SPIR-V instead of x86 code: `--target spirv-unknown-unknown`.
 
 Overall, building your own SPIR-V crate looks like:
 
@@ -61,3 +66,15 @@ To create a GPU crate, look at the [examples/example-shader](examples/example-sh
 
 This is all a little convoluted, hence the [spirv-builder](spirv-builder) crate handles a lot of this.
 
+## Using installed SPIRV-Tools
+
+By default, all of the crates and examples in this repo will compile the `spirv-tools-sys` crate, including a lot of C++ code from [SPIRV-Tools](https://github.com/EmbarkStudios/SPIRV-Tools). If you don't want to build the C++ code because you already have [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools#downloads) installed, or just don't want to spend more time compiling, you can build/run the crate with the `use-installed-tools` feature.
+
+```shell
+cargo run \
+    --manifest-path examples/example-runner/Cargo.toml \
+    --features use-installed-tools \
+    --no-default-features
+```
+
+You should see `warning: use-installed-tools feature on, skipping compilation of C++ code` during the compilation, but otherwise the build will function just the same as if you compiled the C++ code, with the exception that it will fail if you don't have SPIRV-Tools installed correctly.

docs/src/introduction.md

@@ -14,4 +14,3 @@ spirv-std = { path = "../../spirv-std" }
 
 # for parallelism, not really needed though
 rayon = "1.5"
-

examples/example-runner-cpu/Cargo.toml

@@ -5,6 +5,12 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 ash = "0.31"
 ash-window = "0.5"
@@ -16,4 +22,4 @@ winit = "0.23.0"
 ash-molten = { git = "https://github.com/EmbarkStudios/ash-molten", branch = "moltenvk-1.1.0" }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }

examples/example-runner/Cargo.toml

@@ -5,13 +5,19 @@ authors = ["Embark <[email protected]>"]
 edition = "2018"
 license = "MIT OR Apache-2.0"
 
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = ["spirv-builder/use-installed-tools"]
+use-compiled-tools = ["spirv-builder/use-compiled-tools"]
+
 [dependencies]
 wgpu = "0.6.0"
 futures = { version = "0.3", default-features = false, features = ["std", "executor"] }
 winit = { version = "0.22.1", features = ["web-sys"] }
 
 [build-dependencies]
-spirv-builder = { path = "../../spirv-builder" }
+spirv-builder = { path = "../../spirv-builder", default-features = false }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 wgpu-subscriber = "0.1.0"