Start splitting spirv-tools into a compiled vs tool feature set

Author: Jake-Shadle

rustc_codegen_spirv/Cargo.toml

@@ -12,6 +12,11 @@ repository = "https://github.com/EmbarkStudios/rust-gpu"
 [lib]
 crate-type = ["dylib"]
 
+[features]
+# If enabled, uses spirv-tools binaries installed in PATH, instead of
+# compiling and linking the spirv-tools C++ code
+use-installed-tools = ["spirv-tools/use-installed"]
+
 [dependencies]
 bimap = "0.5"
 rspirv = "0.7.0"
@@ -22,13 +27,7 @@ topological-sort = "0.1"
 [dependencies.spirv-tools]
 version = "0.1.0"
 path = "../spirv-tools"
-features = ["opt", "val"]
 
 [dev-dependencies]
 pretty_assertions = "0.6"
 tempfile = "3.1"
-
-[dev-dependencies.spirv-tools]
-version = "0.1.0"
-path = "../spirv-tools"
-features = ["as"]

rustc_codegen_spirv/src/link.rs

@@ -155,41 +155,41 @@ fn link_exe(
 }
 
 fn do_spirv_opt(sess: &Session, spv_binary: Vec<u32>, filename: &Path) -> Vec<u32> {
-    use spirv_tools::{opt, shared};
+    use spirv_tools::{error, opt};
 
-    // This is the same default target environment that spirv-opt uses
-    let mut optimizer = opt::Optimizer::new(shared::TargetEnv::Universal_1_5);
+    let mut optimizer = opt::Optimizer::new(spirv_tools::TargetEnv::default());
 
     optimizer
         .register_size_passes()
         .register_pass(opt::Passes::EliminateDeadConstant)
         .register_pass(opt::Passes::StripDebugInfo);
 
-    let mut opts = opt::Options::new();
+    let result = optimizer.optimize(
+        &spv_binary,
+        &mut |msg: error::Message<'_>| {
+            use error::MessageLevel as Level;
 
-    // We run the validator separately
-    opts.run_validator(false);
+            // TODO: Adds spans here? Not sure how useful with binary, but maybe?
 
-    let mut optimized_binary = None;
+            let mut err = match msg.level {
+                Level::Fatal | Level::InternalError => sess.struct_fatal(&msg.message),
+                Level::Error => sess.struct_err(&msg.message),
+                Level::Warning => sess.struct_warn(&msg.message),
+                Level::Info | Level::Debug => sess.struct_note_without_error(&msg.message),
+            };
 
-    let result = optimizer.optimize(
-        &spv_binary,
-        &mut |data: &[u32]| {
-            optimized_binary = Some(data.to_vec());
+            err.note(&format!("module {:?}", filename));
+            err.emit();
         },
-        Some(&opts),
+        // We currently run the validator separately after optimization or even
+        // if we don't run optimization, the default options don't run the validator
+        None,
     );
 
     match result {
-        Err(opt::RunResult::OptimizerFailed) => {
-            let mut err = sess.struct_warn("spirv-opt failed, leaving as unoptimized");
-            err.note(&format!("module {:?}", filename));
-            spv_binary
-        }
-        Ok(_) => optimized_binary.unwrap(),
+        Ok(binary) => binary.as_ref().to_vec(),
         Err(_) => {
-            let mut err =
-                sess.struct_warn("invalid arguments supplied to spirv-opt, leaving as unoptimized");
+            let mut err = sess.struct_warn("spirv-opt failed, leaving as unoptimized");
             err.note(&format!("module {:?}", filename));
             spv_binary
         }
@@ -199,8 +199,7 @@ fn do_spirv_opt(sess: &Session, spv_binary: Vec<u32>, filename: &Path) -> Vec<u3
 fn do_spirv_val(sess: &Session, spv_binary: &[u32], filename: &Path) {
     use spirv_tools::{shared, val};
 
-    // This is the same default target environment that spirv-val uses
-    let validator = val::Validator::new(shared::TargetEnv::Universal_1_5);
+    let validator = val::Validator::new(spirv_tools::TargetEnv::default());
     let opts = val::ValidatorOptions::default();
 
     if validator.validate(spv_binary, &opts).is_err() {

spirv-tools-sys/Cargo.toml

@@ -7,10 +7,10 @@ build = "build.rs"
 license = "MIT OR Apache-2.0"
 
 [features]
-as = ["tools"]
-opt = ["val"]
-val = ["tools"]
-tools = []
+# Using this feature disables the compilation in the build script, but
+# preserves the types so that spirv-tools can still work without needing
+# to keep copies of some of the basic enums etc
+use-installed = []
 
 [build-dependencies]
 cc = { version = "1.0", features = ["parallel"] }

spirv-tools-sys/build.rs

@@ -219,6 +219,11 @@ fn val(build: &mut Build) {
 }
 
 fn main() {
+    if std::env::var("CARGO_FEATURE_USE_INSTALLED").is_ok() {
+        println!("use-installed feature on, skipping compilation of C++ code");
+        return;
+    }
+
     let mut build = Build::new();
 
     add_includes(&mut build, "spirv-tools", &["", "include"]);
@@ -230,15 +235,8 @@ fn main() {
     );
 
     shared(&mut build);
-
-    // Some opt code requires val as well
-    if cfg!(any(feature = "opt", feature = "val")) {
-        val(&mut build);
-    }
-
-    if cfg!(feature = "opt") {
-        opt(&mut build);
-    }
+    val(&mut build);
+    opt(&mut build);
 
     build.define("SPIRV_CHECK_CONTEXT", None);
 

spirv-tools-sys/src/assembler.rs

@@ -6,12 +6,6 @@ pub enum BinaryOptions {
     PreserveNumberIds = 1 << 1,
 }
 
-#[repr(C)]
-pub struct Binary {
-    pub code: *const u32,
-    pub size: usize,
-}
-
 extern "C" {
     /// Encodes the given SPIR-V assembly text to its binary representation. The
     /// length parameter specifies the number of bytes for text. Encoded binary will
@@ -29,12 +23,7 @@ extern "C" {
         text: *const std::os::raw::c_char,
         size: usize,
         options: u32,
-        binary: *mut *mut Binary,
-        diagnostic: *mut *mut shared::Diagnostic,
-    ) -> crate::shared::SpirvResult;
-
-    /// Frees a binary stream from memory. This is a no-op if binary is a null
-    /// pointer.
-    #[link_name = "spvBinaryDestroy"]
-    pub fn binary_destroy(binary: *mut Binary);
+        binary: *mut *mut shared::Binary,
+        diagnostic: *mut *mut crate::diagnostics::Diagnostic,
+    ) -> shared::SpirvResult;
 }

spirv-tools-sys/src/c/opt.cpp

@@ -1,4 +1,5 @@
 #include "spirv-tools/optimizer.hpp"
+#include <cstring>
 
 struct Optimus;
 
@@ -64,17 +65,13 @@ enum Passes {
     AmdExtToKhr,
 };
 
-enum RunResult {
-    InvalidInputBuffer,
-    InvalidInputSize,
-    InvalidOutputBuffer,
-    InvalidOutputSize,
-    InvalidCallback,
-    OptimizerFailed,
-    OptimizerSucceeded,
-};
-
-typedef void (*optimized_callback)(const uint32_t* data, size_t len, void* ctx);
+typedef void (*message_callback)(
+    spv_message_level_t level,
+    const char* source,
+    const spv_position_t* position,
+    const char* message,
+    void* ctx
+);
 
 extern "C" {
     SPIRV_TOOLS_EXPORT Optimus* optimizer_create(spv_target_env target_env) {
@@ -87,25 +84,47 @@ extern "C" {
         delete (spvtools::Optimizer*)optimizer;
     }
 
-    SPIRV_TOOLS_EXPORT RunResult optimizer_run(
+    SPIRV_TOOLS_EXPORT spv_result_t optimizer_run(
         const Optimus* optimizer,
         const uint32_t* input_ptr,
         size_t input_size,
-        optimized_callback callback,
+        spv_binary* out_binary,
+        message_callback msg_callback,
         void* ctx,
         const spv_optimizer_options options
     ) {
         if (input_ptr == nullptr) {
-            return RunResult::InvalidInputBuffer;
+            return SPV_ERROR_INVALID_POINTER;
         }
 
-        if (callback == nullptr) {
-            return RunResult::InvalidCallback;
+        if (out_binary == nullptr) {
+            return SPV_ERROR_INVALID_POINTER;
         }
 
-        auto output_buff = std::vector<uint32_t>();
-
         auto op = (spvtools::Optimizer*)optimizer;
+
+        if (msg_callback) {
+            op->SetMessageConsumer([msg_callback, ctx](
+                spv_message_level_t level,
+                const char* source,
+                const spv_position_t& position,
+                const char* message) {
+                msg_callback(level, source, &position, message, ctx);
+            });
+        } else {
+            // The optimizer keeps the message consumer as state, so if no
+            // callback is passed to us, we insert a noop callback to ensure
+            // we don't use the state from a previous optimizer run
+            op->SetMessageConsumer([](
+                spv_message_level_t,
+                const char*,
+                const spv_position_t&,
+                const char*)
+                {}
+            );
+        }
+
+        auto output_buff = std::vector<uint32_t>();
         bool success = false;
         if (options == nullptr) {
             success = op->Run(input_ptr, input_size, &output_buff);
@@ -114,12 +133,24 @@ extern "C" {
         }
 
         if (!success) {
-            return RunResult::OptimizerFailed;
+            return SPV_ERROR_INTERNAL;
+        }
+
+        uint32_t* data = new uint32_t[output_buff.size()];
+        if (data == nullptr) {
+            return SPV_ERROR_OUT_OF_MEMORY;
+        }
+
+        spv_binary binary = new spv_binary_t();
+        if (binary == nullptr) {
+            delete[] data;
+            return SPV_ERROR_OUT_OF_MEMORY;
         }
 
-        callback(output_buff.data(), output_buff.size(), ctx);
+        memcpy(data, output_buff.data(), output_buff.size());
+        *out_binary = binary;
 
-        return RunResult::OptimizerSucceeded;
+        return SPV_SUCCESS;
     }
 
     SPIRV_TOOLS_EXPORT void optimizer_register_pass(Optimus* optimizer, Passes pass) {

spirv-tools-sys/src/diagnostics.rs

@@ -0,0 +1,50 @@
+#[repr(C)]
+pub struct Position {
+    pub line: usize,
+    pub column: usize,
+    pub index: usize,
+}
+
+#[repr(C)]
+pub struct Diagnostic {
+    pub position: Position,
+    pub error: *const std::os::raw::c_char,
+    pub is_text_source: bool,
+}
+
+#[derive(Copy, Clone, PartialEq, Debug)]
+#[repr(C)]
+pub enum MessageLevel {
+    /// Unrecoverable error due to environment.
+    /// Will exit the program immediately. E.g.,
+    /// out of memory.
+    Fatal,
+    /// Unrecoverable error due to SPIRV-Tools
+    /// internals.
+    /// Will exit the program immediately. E.g.,
+    /// unimplemented feature.
+    InternalError,
+    /// Normal error due to user input.
+    Error,
+    /// Warning information.
+    Warning,
+    /// General information.
+    Info,
+    /// Debug information.
+    Debug,
+}
+
+pub type MessageCallback = extern "C" fn(
+    MessageLevel,                // level
+    *const std::os::raw::c_char, // source
+    *const Position,             // source position
+    *const std::os::raw::c_char, // the actual message
+    *mut std::ffi::c_void,       // context we use for mapping
+);
+
+extern "C" {
+    /// Destroys a diagnostic object.  This is a no-op if diagnostic is a null
+    /// pointer.
+    #[link_name = "spvDiagnosticDestroy"]
+    pub fn diagnostic_destroy(diag: *mut Diagnostic);
+}

spirv-tools-sys/src/lib.rs

@@ -1,10 +1,5 @@
-pub mod shared;
-
-#[cfg(feature = "opt")]
+pub mod assembler;
+pub mod diagnostics;
 pub mod opt;
-
-#[cfg(feature = "val")]
+pub mod shared;
 pub mod val;
-
-#[cfg(feature = "as")]
-pub mod assembler;

spirv-tools-sys/src/opt.rs

@@ -499,18 +499,6 @@ pub enum Passes {
     AmdExtToKhr,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq)]
-#[repr(C)]
-pub enum RunResult {
-    InvalidInputBuffer,
-    InvalidInputSize,
-    InvalidOutputBuffer,
-    InvalidOutputSize,
-    InvalidCallback,
-    OptimizerFailed,
-    OptimizerSucceeded,
-}
-
 extern "C" {
     pub fn optimizer_create(env: crate::shared::TargetEnv) -> *mut Optimizer;
     pub fn optimizer_destroy(opt: *mut Optimizer);
@@ -519,10 +507,11 @@ extern "C" {
         opt: *const Optimizer,
         input_ptr: *const u32,
         input_size: usize,
-        callback: extern "C" fn(*const u32, usize, *mut std::ffi::c_void),
-        ctx: *mut std::ffi::c_void,
+        binary: *mut *mut crate::shared::Binary,
+        message_callback: crate::diagnostics::MessageCallback,
+        message_ctx: *mut std::ffi::c_void,
         options: *const OptimizerOptions,
-    ) -> RunResult;
+    ) -> crate::shared::SpirvResult;
 
     /// Creates an optimizer options object with default options. Returns a valid
     /// options object. The object remains valid until it is passed into