spirv-fuzz: Improve debugging facilities (KhronosGroup#3074)

Adds an option to run the validator on the SPIR-V binary after each
fuzzer pass has been applied, to help identify when the fuzzer has
made the module invalid.  Also adds a helper method to allow dumping
of the sequence of transformations that have been applied to a JSON
file.

include/spirv-tools/libspirv.h

@@ -627,6 +627,11 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed(
 SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetShrinkerStepLimit(
     spv_fuzzer_options options, uint32_t shrinker_step_limit);
 
+// Enables running the validator after every pass is applied during a fuzzing
+// run.
+SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableFuzzerPassValidation(
+    spv_fuzzer_options options);
+
 // Encodes the given SPIR-V assembly text to its binary representation. The
 // length parameter specifies the number of bytes for text. Encoded binary will
 // be stored into *binary. Any error will be written into *diagnostic if

include/spirv-tools/libspirv.hpp

@@ -229,6 +229,11 @@ class FuzzerOptions {
     spvFuzzerOptionsSetShrinkerStepLimit(options_, shrinker_step_limit);
   }
 
+  // See spvFuzzerOptionsEnableFuzzerPassValidation.
+  void enable_fuzzer_pass_validation() {
+    spvFuzzerOptionsEnableFuzzerPassValidation(options_);
+  }
+
  private:
   spv_fuzzer_options options_;
 };

source/fuzz/fuzzer.cpp

@@ -65,24 +65,54 @@ void MaybeAddPass(
 }  // namespace
 
 struct Fuzzer::Impl {
-  explicit Impl(spv_target_env env) : target_env(env) {}
-
-  const spv_target_env target_env;  // Target environment.
-  MessageConsumer consumer;         // Message consumer.
+  explicit Impl(spv_target_env env, uint32_t random_seed,
+                bool validate_after_each_pass)
+      : target_env(env),
+        seed(random_seed),
+        validate_after_each_fuzzer_pass(validate_after_each_pass) {}
+
+  bool ApplyPassAndCheckValidity(FuzzerPass* pass,
+                                 const opt::IRContext& ir_context,
+                                 const spvtools::SpirvTools& tools) const;
+
+  const spv_target_env target_env;       // Target environment.
+  const uint32_t seed;                   // Seed for random number generator.
+  bool validate_after_each_fuzzer_pass;  // Determines whether the validator
+  // should be invoked after every fuzzer pass.
+  MessageConsumer consumer;  // Message consumer.
 };
 
-Fuzzer::Fuzzer(spv_target_env env) : impl_(MakeUnique<Impl>(env)) {}
+Fuzzer::Fuzzer(spv_target_env env, uint32_t seed,
+               bool validate_after_each_fuzzer_pass)
+    : impl_(MakeUnique<Impl>(env, seed, validate_after_each_fuzzer_pass)) {}
 
 Fuzzer::~Fuzzer() = default;
 
 void Fuzzer::SetMessageConsumer(MessageConsumer c) {
   impl_->consumer = std::move(c);
 }
 
+bool Fuzzer::Impl::ApplyPassAndCheckValidity(
+    FuzzerPass* pass, const opt::IRContext& ir_context,
+    const spvtools::SpirvTools& tools) const {
+  pass->Apply();
+  if (validate_after_each_fuzzer_pass) {
+    std::vector<uint32_t> binary_to_validate;
+    ir_context.module()->ToBinary(&binary_to_validate, false);
+    if (!tools.Validate(&binary_to_validate[0], binary_to_validate.size())) {
+      consumer(SPV_MSG_INFO, nullptr, {},
+               "Binary became invalid during fuzzing (set a breakpoint to "
+               "inspect); stopping.");
+      return false;
+    }
+  }
+  return true;
+}
+
 Fuzzer::FuzzerResultStatus Fuzzer::Run(
     const std::vector<uint32_t>& binary_in,
     const protobufs::FactSequence& initial_facts,
-    spv_const_fuzzer_options options, std::vector<uint32_t>* binary_out,
+    std::vector<uint32_t>* binary_out,
     protobufs::TransformationSequence* transformation_sequence_out) const {
   // Check compatibility between the library version being linked with and the
   // header files being used.
@@ -108,10 +138,8 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run(
       impl_->target_env, impl_->consumer, binary_in.data(), binary_in.size());
   assert(ir_context);
 
-  // Make a PRNG, either from a given seed or from a random device.
-  PseudoRandomGenerator random_generator(
-      options->has_random_seed ? options->random_seed
-                               : static_cast<uint32_t>(std::random_device()()));
+  // Make a PRNG from the seed passed to the fuzzer on creation.
+  PseudoRandomGenerator random_generator(impl_->seed);
 
   // The fuzzer will introduce new ids into the module.  The module's id bound
   // gives the smallest id that can be used for this purpose.  We add an offset
@@ -128,9 +156,13 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run(
 
   // Add some essential ingredients to the module if they are not already
   // present, such as boolean constants.
-  FuzzerPassAddUsefulConstructs(ir_context.get(), &fact_manager,
-                                &fuzzer_context, transformation_sequence_out)
-      .Apply();
+  FuzzerPassAddUsefulConstructs add_useful_constructs(
+      ir_context.get(), &fact_manager, &fuzzer_context,
+      transformation_sequence_out);
+  if (!impl_->ApplyPassAndCheckValidity(&add_useful_constructs, *ir_context,
+                                        tools)) {
+    return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule;
+  }
 
   // Apply some semantics-preserving passes.
   std::vector<std::unique_ptr<FuzzerPass>> passes;
@@ -168,7 +200,11 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run(
          (is_first ||
           fuzzer_context.ChoosePercentage(kChanceOfApplyingAnotherPass))) {
     is_first = false;
-    passes[fuzzer_context.RandomIndex(passes)]->Apply();
+    if (!impl_->ApplyPassAndCheckValidity(
+            passes[fuzzer_context.RandomIndex(passes)].get(), *ir_context,
+            tools)) {
+      return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule;
+    }
   }
 
   // Now apply some passes that it does not make sense to apply repeatedly,
@@ -186,15 +222,13 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run(
   MaybeAddPass<FuzzerPassAdjustSelectionControls>(
       &final_passes, ir_context.get(), &fact_manager, &fuzzer_context,
       transformation_sequence_out);
+  MaybeAddPass<FuzzerPassAddNoContractionDecorations>(
+      &final_passes, ir_context.get(), &fact_manager, &fuzzer_context,
+      transformation_sequence_out);
   for (auto& pass : final_passes) {
-    pass->Apply();
-  }
-
-  if (fuzzer_context.ChooseEven()) {
-    FuzzerPassAddNoContractionDecorations(ir_context.get(), &fact_manager,
-                                          &fuzzer_context,
-                                          transformation_sequence_out)
-        .Apply();
+    if (!impl_->ApplyPassAndCheckValidity(pass.get(), *ir_context, tools)) {
+      return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule;
+    }
   }
 
   // Encode the module as a binary.

source/fuzz/fuzzer.h

@@ -32,11 +32,16 @@ class Fuzzer {
   enum class FuzzerResultStatus {
     kComplete,
     kFailedToCreateSpirvToolsInterface,
+    kFuzzerPassLedToInvalidModule,
     kInitialBinaryInvalid,
   };
 
-  // Constructs a fuzzer from the given target environment.
-  explicit Fuzzer(spv_target_env env);
+  // Constructs a fuzzer from the given target environment |env|.  |seed| is a
+  // seed for pseudo-random number generation.
+  // |validate_after_each_fuzzer_pass| controls whether the validator will be
+  // invoked after every fuzzer pass is applied.
+  explicit Fuzzer(spv_target_env env, uint32_t seed,
+                  bool validate_after_each_fuzzer_pass);
 
   // Disables copy/move constructor/assignment operations.
   Fuzzer(const Fuzzer&) = delete;
@@ -51,14 +56,13 @@ class Fuzzer {
   void SetMessageConsumer(MessageConsumer consumer);
 
   // Transforms |binary_in| to |binary_out| by running a number of randomized
-  // fuzzer passes, controlled via |options|.  Initial facts about the input
-  // binary and the context in which it will execute are provided via
-  // |initial_facts|.  The transformation sequence that was applied is returned
-  // via |transformation_sequence_out|.
+  // fuzzer passes.  Initial facts about the input binary and the context in
+  // which it will execute are provided via |initial_facts|.  The transformation
+  // sequence that was applied is returned via |transformation_sequence_out|.
   FuzzerResultStatus Run(
       const std::vector<uint32_t>& binary_in,
       const protobufs::FactSequence& initial_facts,
-      spv_const_fuzzer_options options, std::vector<uint32_t>* binary_out,
+      std::vector<uint32_t>* binary_out,
       protobufs::TransformationSequence* transformation_sequence_out) const;
 
  private:

source/spirv_fuzzer_options.cpp

@@ -23,7 +23,8 @@ spv_fuzzer_options_t::spv_fuzzer_options_t()
     : has_random_seed(false),
       random_seed(0),
       replay_validation_enabled(false),
-      shrinker_step_limit(kDefaultStepLimit) {}
+      shrinker_step_limit(kDefaultStepLimit),
+      fuzzer_pass_validation_enabled(false) {}
 
 SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate() {
   return new spv_fuzzer_options_t();
@@ -48,3 +49,8 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetShrinkerStepLimit(
     spv_fuzzer_options options, uint32_t shrinker_step_limit) {
   options->shrinker_step_limit = shrinker_step_limit;
 }
+
+SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableFuzzerPassValidation(
+    spv_fuzzer_options options) {
+  options->fuzzer_pass_validation_enabled = true;
+}

source/spirv_fuzzer_options.h

@@ -34,6 +34,9 @@ struct spv_fuzzer_options_t {
 
   // See spvFuzzerOptionsSetShrinkerStepLimit.
   uint32_t shrinker_step_limit;
+
+  // See spvFuzzerOptionsValidateAfterEveryPass.
+  bool fuzzer_pass_validation_enabled;
 };
 
 #endif  // SOURCE_SPIRV_FUZZER_OPTIONS_H_

test/fuzz/fuzz_test_util.cpp

@@ -14,6 +14,7 @@
 
 #include "test/fuzz/fuzz_test_util.h"
 
+#include <fstream>
 #include <iostream>
 
 #include "tools/io.h"
@@ -105,5 +106,20 @@ void DumpShader(const std::vector<uint32_t>& binary, const char* filename) {
   }
 }
 
+void DumpTransformationsJson(
+    const protobufs::TransformationSequence& transformations,
+    const char* filename) {
+  std::string json_string;
+  auto json_options = google::protobuf::util::JsonOptions();
+  json_options.add_whitespace = true;
+  auto json_generation_status = google::protobuf::util::MessageToJsonString(
+      transformations, &json_string, json_options);
+  if (json_generation_status == google::protobuf::util::Status::OK) {
+    std::ofstream transformations_json_file(filename);
+    transformations_json_file << json_string;
+    transformations_json_file.close();
+  }
+}
+
 }  // namespace fuzz
 }  // namespace spvtools

test/fuzz/fuzz_test_util.h

@@ -19,6 +19,7 @@
 
 #include <vector>
 
+#include "source/fuzz/protobufs/spirvfuzz_protobufs.h"
 #include "source/opt/build_module.h"
 #include "source/opt/ir_context.h"
 #include "spirv-tools/libspirv.h"
@@ -100,6 +101,12 @@ void DumpShader(opt::IRContext* context, const char* filename);
 // Dumps |binary| to file |filename|. Useful for interactive debugging.
 void DumpShader(const std::vector<uint32_t>& binary, const char* filename);
 
+// Dumps |transformations| to file |filename| in JSON format. Useful for
+// interactive debugging.
+void DumpTransformationsJson(
+    const protobufs::TransformationSequence& transformations,
+    const char* filename);
+
 }  // namespace fuzz
 }  // namespace spvtools
 

test/fuzz/fuzzer_replayer_test.cpp

@@ -62,13 +62,11 @@ void RunFuzzerAndReplayer(const std::string& shader,
   for (uint32_t seed = initial_seed; seed < initial_seed + num_runs; seed++) {
     std::vector<uint32_t> fuzzer_binary_out;
     protobufs::TransformationSequence fuzzer_transformation_sequence_out;
-    spvtools::FuzzerOptions fuzzer_options;
-    spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed);
 
-    Fuzzer fuzzer(env);
+    Fuzzer fuzzer(env, seed, true);
     fuzzer.SetMessageConsumer(kSilentConsumer);
     auto fuzzer_result_status =
-        fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out,
+        fuzzer.Run(binary_in, initial_facts, &fuzzer_binary_out,
                    &fuzzer_transformation_sequence_out);
     ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status);
     ASSERT_TRUE(t.Validate(fuzzer_binary_out));

test/fuzz/fuzzer_shrinker_test.cpp

@@ -165,12 +165,10 @@ void RunFuzzerAndShrinker(const std::string& shader,
   // Run the fuzzer and check that it successfully yields a valid binary.
   std::vector<uint32_t> fuzzer_binary_out;
   protobufs::TransformationSequence fuzzer_transformation_sequence_out;
-  spvtools::FuzzerOptions fuzzer_options;
-  spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed);
-  Fuzzer fuzzer(env);
+  Fuzzer fuzzer(env, seed, true);
   fuzzer.SetMessageConsumer(kSilentConsumer);
   auto fuzzer_result_status =
-      fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out,
+      fuzzer.Run(binary_in, initial_facts, &fuzzer_binary_out,
                  &fuzzer_transformation_sequence_out);
   ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status);
   ASSERT_TRUE(t.Validate(fuzzer_binary_out));

tools/fuzz/fuzz.cpp

@@ -17,6 +17,7 @@
 #include <cstring>
 #include <fstream>
 #include <functional>
+#include <random>
 #include <sstream>
 #include <string>
 
@@ -106,6 +107,10 @@ Options (in lexicographical order):
                facts to make the guard non-obviously false.  This option is a
                helper for massaging crash-inducing tests into a runnable
                format; it does not perform any fuzzing.
+  --fuzzer-pass-validation
+               Run the validator after applying each fuzzer pass during
+               fuzzing.  Aborts fuzzing early if an invalid binary is created.
+               Useful for debugging spirv-fuzz.
   --replay
                File from which to read a sequence of transformations to replay
                (instead of fuzzing)
@@ -179,6 +184,9 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file,
       } else if (0 == strncmp(cur_arg, "--force-render-red",
                               sizeof("--force-render-red") - 1)) {
         force_render_red = true;
+      } else if (0 == strncmp(cur_arg, "--fuzzer-pass-validation",
+                              sizeof("--fuzzer-pass-validation") - 1)) {
+        fuzzer_options->enable_fuzzer_pass_validation();
       } else if (0 == strncmp(cur_arg, "--replay=", sizeof("--replay=") - 1)) {
         const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg);
         *replay_transformations_file = std::string(split_flag.second);
@@ -397,16 +405,21 @@ bool Shrink(const spv_target_env& target_env,
 }
 
 bool Fuzz(const spv_target_env& target_env,
-          const spvtools::FuzzerOptions& fuzzer_options,
+          spv_const_fuzzer_options fuzzer_options,
           const std::vector<uint32_t>& binary_in,
           const spvtools::fuzz::protobufs::FactSequence& initial_facts,
           std::vector<uint32_t>* binary_out,
           spvtools::fuzz::protobufs::TransformationSequence*
               transformations_applied) {
-  spvtools::fuzz::Fuzzer fuzzer(target_env);
+  spvtools::fuzz::Fuzzer fuzzer(
+      target_env,
+      fuzzer_options->has_random_seed
+          ? fuzzer_options->random_seed
+          : static_cast<uint32_t>(std::random_device()()),
+      fuzzer_options->fuzzer_pass_validation_enabled);
   fuzzer.SetMessageConsumer(spvtools::utils::CLIMessageConsumer);
-  auto fuzz_result_status = fuzzer.Run(binary_in, initial_facts, fuzzer_options,
-                                       binary_out, transformations_applied);
+  auto fuzz_result_status =
+      fuzzer.Run(binary_in, initial_facts, binary_out, transformations_applied);
   if (fuzz_result_status !=
       spvtools::fuzz::Fuzzer::FuzzerResultStatus::kComplete) {
     spvtools::Error(FuzzDiagnostic, nullptr, {}, "Error running fuzzer");