From 52e9cc93016445b6e6b567d7cd696b23570b2513 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Wed, 27 Nov 2019 18:05:56 +0000 Subject: [PATCH] spirv-fuzz: Improve debugging facilities (#3074) Adds an option to run the validator on the SPIR-V binary after each fuzzer pass has been applied, to help identify when the fuzzer has made the module invalid. Also adds a helper method to allow dumping of the sequence of transformations that have been applied to a JSON file. --- include/spirv-tools/libspirv.h | 5 ++ include/spirv-tools/libspirv.hpp | 5 ++ source/fuzz/fuzzer.cpp | 78 +++++++++++++++++++++--------- source/fuzz/fuzzer.h | 18 ++++--- source/spirv_fuzzer_options.cpp | 8 ++- source/spirv_fuzzer_options.h | 3 ++ test/fuzz/fuzz_test_util.cpp | 16 ++++++ test/fuzz/fuzz_test_util.h | 7 +++ test/fuzz/fuzzer_replayer_test.cpp | 6 +-- test/fuzz/fuzzer_shrinker_test.cpp | 6 +-- tools/fuzz/fuzz.cpp | 21 ++++++-- 11 files changed, 131 insertions(+), 42 deletions(-) diff --git a/include/spirv-tools/libspirv.h b/include/spirv-tools/libspirv.h index dd2526bf3b..2f4c7d60bc 100644 --- a/include/spirv-tools/libspirv.h +++ b/include/spirv-tools/libspirv.h @@ -627,6 +627,11 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed( SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetShrinkerStepLimit( spv_fuzzer_options options, uint32_t shrinker_step_limit); +// Enables running the validator after every pass is applied during a fuzzing +// run. +SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableFuzzerPassValidation( + spv_fuzzer_options options); + // Encodes the given SPIR-V assembly text to its binary representation. The // length parameter specifies the number of bytes for text. Encoded binary will // be stored into *binary. Any error will be written into *diagnostic if diff --git a/include/spirv-tools/libspirv.hpp b/include/spirv-tools/libspirv.hpp index b09fdd457d..ceadef8fb2 100644 --- a/include/spirv-tools/libspirv.hpp +++ b/include/spirv-tools/libspirv.hpp @@ -229,6 +229,11 @@ class FuzzerOptions { spvFuzzerOptionsSetShrinkerStepLimit(options_, shrinker_step_limit); } + // See spvFuzzerOptionsEnableFuzzerPassValidation. + void enable_fuzzer_pass_validation() { + spvFuzzerOptionsEnableFuzzerPassValidation(options_); + } + private: spv_fuzzer_options options_; }; diff --git a/source/fuzz/fuzzer.cpp b/source/fuzz/fuzzer.cpp index 01b4258d37..20e714d7cc 100644 --- a/source/fuzz/fuzzer.cpp +++ b/source/fuzz/fuzzer.cpp @@ -65,13 +65,26 @@ void MaybeAddPass( } // namespace struct Fuzzer::Impl { - explicit Impl(spv_target_env env) : target_env(env) {} - - const spv_target_env target_env; // Target environment. - MessageConsumer consumer; // Message consumer. + explicit Impl(spv_target_env env, uint32_t random_seed, + bool validate_after_each_pass) + : target_env(env), + seed(random_seed), + validate_after_each_fuzzer_pass(validate_after_each_pass) {} + + bool ApplyPassAndCheckValidity(FuzzerPass* pass, + const opt::IRContext& ir_context, + const spvtools::SpirvTools& tools) const; + + const spv_target_env target_env; // Target environment. + const uint32_t seed; // Seed for random number generator. + bool validate_after_each_fuzzer_pass; // Determines whether the validator + // should be invoked after every fuzzer pass. + MessageConsumer consumer; // Message consumer. }; -Fuzzer::Fuzzer(spv_target_env env) : impl_(MakeUnique(env)) {} +Fuzzer::Fuzzer(spv_target_env env, uint32_t seed, + bool validate_after_each_fuzzer_pass) + : impl_(MakeUnique(env, seed, validate_after_each_fuzzer_pass)) {} Fuzzer::~Fuzzer() = default; @@ -79,10 +92,27 @@ void Fuzzer::SetMessageConsumer(MessageConsumer c) { impl_->consumer = std::move(c); } +bool Fuzzer::Impl::ApplyPassAndCheckValidity( + FuzzerPass* pass, const opt::IRContext& ir_context, + const spvtools::SpirvTools& tools) const { + pass->Apply(); + if (validate_after_each_fuzzer_pass) { + std::vector binary_to_validate; + ir_context.module()->ToBinary(&binary_to_validate, false); + if (!tools.Validate(&binary_to_validate[0], binary_to_validate.size())) { + consumer(SPV_MSG_INFO, nullptr, {}, + "Binary became invalid during fuzzing (set a breakpoint to " + "inspect); stopping."); + return false; + } + } + return true; +} + Fuzzer::FuzzerResultStatus Fuzzer::Run( const std::vector& binary_in, const protobufs::FactSequence& initial_facts, - spv_const_fuzzer_options options, std::vector* binary_out, + std::vector* binary_out, protobufs::TransformationSequence* transformation_sequence_out) const { // Check compatibility between the library version being linked with and the // header files being used. @@ -108,10 +138,8 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run( impl_->target_env, impl_->consumer, binary_in.data(), binary_in.size()); assert(ir_context); - // Make a PRNG, either from a given seed or from a random device. - PseudoRandomGenerator random_generator( - options->has_random_seed ? options->random_seed - : static_cast(std::random_device()())); + // Make a PRNG from the seed passed to the fuzzer on creation. + PseudoRandomGenerator random_generator(impl_->seed); // The fuzzer will introduce new ids into the module. The module's id bound // gives the smallest id that can be used for this purpose. We add an offset @@ -128,9 +156,13 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run( // Add some essential ingredients to the module if they are not already // present, such as boolean constants. - FuzzerPassAddUsefulConstructs(ir_context.get(), &fact_manager, - &fuzzer_context, transformation_sequence_out) - .Apply(); + FuzzerPassAddUsefulConstructs add_useful_constructs( + ir_context.get(), &fact_manager, &fuzzer_context, + transformation_sequence_out); + if (!impl_->ApplyPassAndCheckValidity(&add_useful_constructs, *ir_context, + tools)) { + return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule; + } // Apply some semantics-preserving passes. std::vector> passes; @@ -168,7 +200,11 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run( (is_first || fuzzer_context.ChoosePercentage(kChanceOfApplyingAnotherPass))) { is_first = false; - passes[fuzzer_context.RandomIndex(passes)]->Apply(); + if (!impl_->ApplyPassAndCheckValidity( + passes[fuzzer_context.RandomIndex(passes)].get(), *ir_context, + tools)) { + return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule; + } } // Now apply some passes that it does not make sense to apply repeatedly, @@ -186,15 +222,13 @@ Fuzzer::FuzzerResultStatus Fuzzer::Run( MaybeAddPass( &final_passes, ir_context.get(), &fact_manager, &fuzzer_context, transformation_sequence_out); + MaybeAddPass( + &final_passes, ir_context.get(), &fact_manager, &fuzzer_context, + transformation_sequence_out); for (auto& pass : final_passes) { - pass->Apply(); - } - - if (fuzzer_context.ChooseEven()) { - FuzzerPassAddNoContractionDecorations(ir_context.get(), &fact_manager, - &fuzzer_context, - transformation_sequence_out) - .Apply(); + if (!impl_->ApplyPassAndCheckValidity(pass.get(), *ir_context, tools)) { + return Fuzzer::FuzzerResultStatus::kFuzzerPassLedToInvalidModule; + } } // Encode the module as a binary. diff --git a/source/fuzz/fuzzer.h b/source/fuzz/fuzzer.h index a257c2a19a..c1d2deeef2 100644 --- a/source/fuzz/fuzzer.h +++ b/source/fuzz/fuzzer.h @@ -32,11 +32,16 @@ class Fuzzer { enum class FuzzerResultStatus { kComplete, kFailedToCreateSpirvToolsInterface, + kFuzzerPassLedToInvalidModule, kInitialBinaryInvalid, }; - // Constructs a fuzzer from the given target environment. - explicit Fuzzer(spv_target_env env); + // Constructs a fuzzer from the given target environment |env|. |seed| is a + // seed for pseudo-random number generation. + // |validate_after_each_fuzzer_pass| controls whether the validator will be + // invoked after every fuzzer pass is applied. + explicit Fuzzer(spv_target_env env, uint32_t seed, + bool validate_after_each_fuzzer_pass); // Disables copy/move constructor/assignment operations. Fuzzer(const Fuzzer&) = delete; @@ -51,14 +56,13 @@ class Fuzzer { void SetMessageConsumer(MessageConsumer consumer); // Transforms |binary_in| to |binary_out| by running a number of randomized - // fuzzer passes, controlled via |options|. Initial facts about the input - // binary and the context in which it will execute are provided via - // |initial_facts|. The transformation sequence that was applied is returned - // via |transformation_sequence_out|. + // fuzzer passes. Initial facts about the input binary and the context in + // which it will execute are provided via |initial_facts|. The transformation + // sequence that was applied is returned via |transformation_sequence_out|. FuzzerResultStatus Run( const std::vector& binary_in, const protobufs::FactSequence& initial_facts, - spv_const_fuzzer_options options, std::vector* binary_out, + std::vector* binary_out, protobufs::TransformationSequence* transformation_sequence_out) const; private: diff --git a/source/spirv_fuzzer_options.cpp b/source/spirv_fuzzer_options.cpp index ab8903e1eb..b407f148a7 100644 --- a/source/spirv_fuzzer_options.cpp +++ b/source/spirv_fuzzer_options.cpp @@ -23,7 +23,8 @@ spv_fuzzer_options_t::spv_fuzzer_options_t() : has_random_seed(false), random_seed(0), replay_validation_enabled(false), - shrinker_step_limit(kDefaultStepLimit) {} + shrinker_step_limit(kDefaultStepLimit), + fuzzer_pass_validation_enabled(false) {} SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate() { return new spv_fuzzer_options_t(); @@ -48,3 +49,8 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetShrinkerStepLimit( spv_fuzzer_options options, uint32_t shrinker_step_limit) { options->shrinker_step_limit = shrinker_step_limit; } + +SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableFuzzerPassValidation( + spv_fuzzer_options options) { + options->fuzzer_pass_validation_enabled = true; +} diff --git a/source/spirv_fuzzer_options.h b/source/spirv_fuzzer_options.h index 7bb16c7387..143f77f84e 100644 --- a/source/spirv_fuzzer_options.h +++ b/source/spirv_fuzzer_options.h @@ -34,6 +34,9 @@ struct spv_fuzzer_options_t { // See spvFuzzerOptionsSetShrinkerStepLimit. uint32_t shrinker_step_limit; + + // See spvFuzzerOptionsValidateAfterEveryPass. + bool fuzzer_pass_validation_enabled; }; #endif // SOURCE_SPIRV_FUZZER_OPTIONS_H_ diff --git a/test/fuzz/fuzz_test_util.cpp b/test/fuzz/fuzz_test_util.cpp index bc6d4ee6e9..1d15ad6248 100644 --- a/test/fuzz/fuzz_test_util.cpp +++ b/test/fuzz/fuzz_test_util.cpp @@ -14,6 +14,7 @@ #include "test/fuzz/fuzz_test_util.h" +#include #include #include "tools/io.h" @@ -105,5 +106,20 @@ void DumpShader(const std::vector& binary, const char* filename) { } } +void DumpTransformationsJson( + const protobufs::TransformationSequence& transformations, + const char* filename) { + std::string json_string; + auto json_options = google::protobuf::util::JsonOptions(); + json_options.add_whitespace = true; + auto json_generation_status = google::protobuf::util::MessageToJsonString( + transformations, &json_string, json_options); + if (json_generation_status == google::protobuf::util::Status::OK) { + std::ofstream transformations_json_file(filename); + transformations_json_file << json_string; + transformations_json_file.close(); + } +} + } // namespace fuzz } // namespace spvtools diff --git a/test/fuzz/fuzz_test_util.h b/test/fuzz/fuzz_test_util.h index 93f37e5678..9e08bf6891 100644 --- a/test/fuzz/fuzz_test_util.h +++ b/test/fuzz/fuzz_test_util.h @@ -19,6 +19,7 @@ #include +#include "source/fuzz/protobufs/spirvfuzz_protobufs.h" #include "source/opt/build_module.h" #include "source/opt/ir_context.h" #include "spirv-tools/libspirv.h" @@ -100,6 +101,12 @@ void DumpShader(opt::IRContext* context, const char* filename); // Dumps |binary| to file |filename|. Useful for interactive debugging. void DumpShader(const std::vector& binary, const char* filename); +// Dumps |transformations| to file |filename| in JSON format. Useful for +// interactive debugging. +void DumpTransformationsJson( + const protobufs::TransformationSequence& transformations, + const char* filename); + } // namespace fuzz } // namespace spvtools diff --git a/test/fuzz/fuzzer_replayer_test.cpp b/test/fuzz/fuzzer_replayer_test.cpp index 22a26fc6b0..17998c5a73 100644 --- a/test/fuzz/fuzzer_replayer_test.cpp +++ b/test/fuzz/fuzzer_replayer_test.cpp @@ -62,13 +62,11 @@ void RunFuzzerAndReplayer(const std::string& shader, for (uint32_t seed = initial_seed; seed < initial_seed + num_runs; seed++) { std::vector fuzzer_binary_out; protobufs::TransformationSequence fuzzer_transformation_sequence_out; - spvtools::FuzzerOptions fuzzer_options; - spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed); - Fuzzer fuzzer(env); + Fuzzer fuzzer(env, seed, true); fuzzer.SetMessageConsumer(kSilentConsumer); auto fuzzer_result_status = - fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out, + fuzzer.Run(binary_in, initial_facts, &fuzzer_binary_out, &fuzzer_transformation_sequence_out); ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status); ASSERT_TRUE(t.Validate(fuzzer_binary_out)); diff --git a/test/fuzz/fuzzer_shrinker_test.cpp b/test/fuzz/fuzzer_shrinker_test.cpp index 648507031a..8b72e3c531 100644 --- a/test/fuzz/fuzzer_shrinker_test.cpp +++ b/test/fuzz/fuzzer_shrinker_test.cpp @@ -165,12 +165,10 @@ void RunFuzzerAndShrinker(const std::string& shader, // Run the fuzzer and check that it successfully yields a valid binary. std::vector fuzzer_binary_out; protobufs::TransformationSequence fuzzer_transformation_sequence_out; - spvtools::FuzzerOptions fuzzer_options; - spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed); - Fuzzer fuzzer(env); + Fuzzer fuzzer(env, seed, true); fuzzer.SetMessageConsumer(kSilentConsumer); auto fuzzer_result_status = - fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out, + fuzzer.Run(binary_in, initial_facts, &fuzzer_binary_out, &fuzzer_transformation_sequence_out); ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status); ASSERT_TRUE(t.Validate(fuzzer_binary_out)); diff --git a/tools/fuzz/fuzz.cpp b/tools/fuzz/fuzz.cpp index 5d582f5ccb..cb98914fac 100644 --- a/tools/fuzz/fuzz.cpp +++ b/tools/fuzz/fuzz.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -106,6 +107,10 @@ Options (in lexicographical order): facts to make the guard non-obviously false. This option is a helper for massaging crash-inducing tests into a runnable format; it does not perform any fuzzing. + --fuzzer-pass-validation + Run the validator after applying each fuzzer pass during + fuzzing. Aborts fuzzing early if an invalid binary is created. + Useful for debugging spirv-fuzz. --replay File from which to read a sequence of transformations to replay (instead of fuzzing) @@ -179,6 +184,9 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file, } else if (0 == strncmp(cur_arg, "--force-render-red", sizeof("--force-render-red") - 1)) { force_render_red = true; + } else if (0 == strncmp(cur_arg, "--fuzzer-pass-validation", + sizeof("--fuzzer-pass-validation") - 1)) { + fuzzer_options->enable_fuzzer_pass_validation(); } else if (0 == strncmp(cur_arg, "--replay=", sizeof("--replay=") - 1)) { const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg); *replay_transformations_file = std::string(split_flag.second); @@ -397,16 +405,21 @@ bool Shrink(const spv_target_env& target_env, } bool Fuzz(const spv_target_env& target_env, - const spvtools::FuzzerOptions& fuzzer_options, + spv_const_fuzzer_options fuzzer_options, const std::vector& binary_in, const spvtools::fuzz::protobufs::FactSequence& initial_facts, std::vector* binary_out, spvtools::fuzz::protobufs::TransformationSequence* transformations_applied) { - spvtools::fuzz::Fuzzer fuzzer(target_env); + spvtools::fuzz::Fuzzer fuzzer( + target_env, + fuzzer_options->has_random_seed + ? fuzzer_options->random_seed + : static_cast(std::random_device()()), + fuzzer_options->fuzzer_pass_validation_enabled); fuzzer.SetMessageConsumer(spvtools::utils::CLIMessageConsumer); - auto fuzz_result_status = fuzzer.Run(binary_in, initial_facts, fuzzer_options, - binary_out, transformations_applied); + auto fuzz_result_status = + fuzzer.Run(binary_in, initial_facts, binary_out, transformations_applied); if (fuzz_result_status != spvtools::fuzz::Fuzzer::FuzzerResultStatus::kComplete) { spvtools::Error(FuzzDiagnostic, nullptr, {}, "Error running fuzzer");